diff options
625 files changed, 21535 insertions, 11713 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 9b0e2dbd2e..c4aae3a50a 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -2,6 +2,7 @@ env: # Global defaults PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y" MAKEJOBS: "-j10" TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache + CI_FAILFAST_TEST_LEAVE_DANGLING: "1" # Cirrus CI does not care about dangling process and setting this variable avoids killing the CI script itself on error CCACHE_SIZE: "200M" CCACHE_DIR: "/tmp/ccache_dir" CCACHE_NOHASHDIR: "1" # Debug info might contain a stale path if the build dir changes, but this is fine @@ -72,6 +73,19 @@ task: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV task: + name: 'tidy [jammy]' + << : *GLOBAL_TASK_TEMPLATE + container: + image: ubuntu:jammy + cpu: 2 + memory: 5G + # For faster CI feedback, immediately schedule the linters + << : *CREDITS_TEMPLATE + env: + << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + FILE_ENV: "./ci/test/00_setup_env_native_tidy.sh" + +task: name: "Win64 native [msvc]" << : *FILTER_TEMPLATE windows_container: @@ -82,14 +96,17 @@ task: env: PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Current\Bin;%PATH%' PYTHONUTF8: 1 - CI_VCPKG_TAG: '2021.05.12' + CI_VCPKG_TAG: '2022.04.12' VCPKG_DOWNLOADS: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\downloads' VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' - QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.2/single/qt-everywhere-src-5.15.2.zip' - QT_LOCAL_PATH: 'C:\qt-everywhere-src-5.15.2.zip' - QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.2' + CCACHE_DIR: 'C:\Users\ContainerAdministrator\AppData\Local\ccache' + WRAPPED_CL: 'C:\Users\ContainerAdministrator\AppData\Local\Temp\cirrus-ci-build\ci\test\wrapped-cl.bat' + QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.3/single/qt-everywhere-opensource-src-5.15.3.zip' + QT_LOCAL_PATH: 'C:\qt-everywhere-opensource-src-5.15.3.zip' + QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.3' QTBASEDIR: 'C:\Qt_static' x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"' + QT_CONFIGURE_COMMAND: '..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml' IgnoreWarnIntDirInTempDetected: 'true' merge_script: - git config --global user.email "ci@ci.ci" @@ -104,7 +121,7 @@ task: folder: "%QTBASEDIR%" reupload_on_changes: false fingerprint_script: - - echo %QT_DOWNLOAD_URL% + - echo %QT_DOWNLOAD_URL% %QT_CONFIGURE_COMMAND% - msbuild -version populate_script: - curl -L -o C:\jom.zip http://download.qt.io/official_releases/jom/jom.zip @@ -116,7 +133,7 @@ task: - cd %QT_SOURCE_DIR% - mkdir build - cd build - - ..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml -prefix %QTBASEDIR% + - '%QT_CONFIGURE_COMMAND% -prefix %QTBASEDIR%' - jom - jom install vcpkg_tools_cache: @@ -130,12 +147,17 @@ task: reupload_on_changes: true fingerprint_script: - echo %CI_VCPKG_TAG% + - type build_msvc\vcpkg.json - msbuild -version populate_script: - mkdir %VCPKG_DEFAULT_BINARY_CACHE% - install_python_script: + ccache_cache: + folder: '%CCACHE_DIR%' + install_tools_script: + - choco install --yes --no-progress ccache - choco install --yes --no-progress python3 --version=3.9.6 - pip install zmq + - ccache --version - python -VV install_vcpkg_script: - cd .. @@ -147,9 +169,12 @@ task: - .\vcpkg integrate install - .\vcpkg version build_script: + - '%x64_NATIVE_TOOLS%' - cd %CIRRUS_WORKING_DIR% + - ccache --zero-stats - python build_msvc\msvc-autogen.py - - msbuild build_msvc\bitcoin.sln -property:Configuration=Release -maxCpuCount -verbosity:minimal -noLogo + - msbuild build_msvc\bitcoin.sln -property:CLToolExe=%WRAPPED_CL% -property:Configuration=Release -maxCpuCount -verbosity:minimal -noLogo + - ccache --show-stats unit_tests_script: - src\test_bitcoin.exe -l test_suite - src\bench_bitcoin.exe > NUL @@ -195,7 +220,7 @@ task: FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh" task: - name: '[previous releases, uses qt5 dev package and some depends packages, DEBUG] [unsigned char] [bionic]' + name: '[previous releases, uses qt5 dev package and some depends packages, DEBUG] [unsigned char] [buster]' previous_releases_cache: folder: "releases" << : *GLOBAL_TASK_TEMPLATE @@ -258,13 +283,13 @@ task: FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh" task: - name: '[no wallet] [bionic]' + name: '[no wallet, libbitcoinkernel] [bionic]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:bionic env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_native_nowallet.sh" + FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh" task: name: 'macOS 10.15 [gui, no tests] [focal]' @@ -287,7 +312,7 @@ task: << : *GLOBAL_TASK_TEMPLATE macos_instance: # Use latest image, but hardcode version to avoid silent upgrades (and breaks) - image: monterey-xcode-13.2 # https://cirrus-ci.org/guide/macOS + image: monterey-xcode-13.3 # https://cirrus-ci.org/guide/macOS env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV CI_USE_APT_INSTALL: "no" diff --git a/.gitignore b/.gitignore index f84a53178e..6c888bfdc4 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ src/bitcoin-gui src/bitcoin-node src/bitcoin-tx src/bitcoin-util +src/bitcoin-chainstate src/bitcoin-wallet src/test/fuzz/fuzz src/test/test_bitcoin @@ -76,7 +77,6 @@ src/qt/bitcoin-qt.includes *.log *.trs *.dmg -*.iso *.json.h *.raw.h @@ -150,3 +150,5 @@ osx_volname dist/ /guix-build-* + +/ci/scratch/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 254e610393..aec6995d3b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,10 +10,9 @@ First, in terms of structure, there is no particular concept of "Bitcoin Core developers" in the sense of privileged people. Open source often naturally revolves around a meritocracy where contributors earn trust from the developer community over time. Nevertheless, some hierarchy is necessary for practical -purposes. As such, there are repository "maintainers" who are responsible for -merging pull requests, as well as a "lead maintainer" who is responsible for the -[release cycle](/doc/release-process.md) as well as overall merging, moderation -and appointment of maintainers. +purposes. As such, there are repository maintainers who are responsible for +merging pull requests, the [release cycle](/doc/release-process.md), and +moderation. Getting Started --------------- @@ -153,7 +152,8 @@ the pull request affects. Valid areas as: - `test`, `qa` or `ci` for changes to the unit tests, QA tests or CI code - `util` or `lib` for changes to the utils or libraries - `wallet` for changes to the wallet code - - `build` for changes to the GNU Autotools or reproducible builds + - `build` for changes to the GNU Autotools or MSVC builds + - `guix` for changes to the GUIX reproducible builds Examples: @@ -292,7 +292,7 @@ projects such as libsecp256k1), and is not to be confused with overall Bitcoin Network Protocol consensus changes. Whether a pull request is merged into Bitcoin Core rests with the project merge -maintainers and ultimately the project lead. +maintainers. Maintainers will take into consideration if a patch is in line with the general principles of the project; meets the minimum standards for inclusion; and will diff --git a/Makefile.am b/Makefile.am index 1412624d54..05e89f12b7 100644 --- a/Makefile.am +++ b/Makefile.am @@ -12,7 +12,7 @@ if ENABLE_MAN SUBDIRS += doc/man endif .PHONY: deploy FORCE -.INTERMEDIATE: $(OSX_TEMP_ISO) $(COVERAGE_INFO) +.INTERMEDIATE: $(COVERAGE_INFO) export PYTHONPATH @@ -37,8 +37,6 @@ space := $(empty) $(empty) OSX_APP=Bitcoin-Qt.app OSX_VOLNAME = $(subst $(space),-,$(PACKAGE_NAME)) OSX_DMG = $(OSX_VOLNAME).dmg -OSX_TEMP_ISO = $(OSX_DMG:.dmg=).temp.iso -OSX_BACKGROUND_IMAGE=$(top_srcdir)/contrib/macdeploy/background.tiff OSX_DEPLOY_SCRIPT=$(top_srcdir)/contrib/macdeploy/macdeployqtplus OSX_INSTALLER_ICONS=$(top_srcdir)/src/qt/res/icons/bitcoin.icns OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed @@ -48,7 +46,8 @@ DIST_CONTRIB = \ $(top_srcdir)/test/sanitizer_suppressions/tsan \ $(top_srcdir)/test/sanitizer_suppressions/ubsan \ $(top_srcdir)/contrib/linearize/linearize-data.py \ - $(top_srcdir)/contrib/linearize/linearize-hashes.py + $(top_srcdir)/contrib/linearize/linearize-hashes.py \ + $(top_srcdir)/contrib/signet/miner DIST_SHARE = \ $(top_srcdir)/share/genbuild.sh \ @@ -64,7 +63,6 @@ WINDOWS_PACKAGING = $(top_srcdir)/share/pixmaps/bitcoin.ico \ $(top_srcdir)/doc/README_windows.txt OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_INSTALLER_ICONS) \ - $(top_srcdir)/contrib/macdeploy/detached-sig-apply.sh \ $(top_srcdir)/contrib/macdeploy/detached-sig-create.sh COVERAGE_INFO = $(COV_TOOL_WRAPPER) baseline.info \ @@ -130,31 +128,16 @@ $(OSX_DMG): $(OSX_APP_BUILT) $(OSX_PACKAGING) deploydir: $(OSX_DMG) else !BUILD_DARWIN APP_DIST_DIR=$(top_builddir)/dist -APP_DIST_EXTRAS=$(APP_DIST_DIR)/.background/background.tiff $(APP_DIST_DIR)/.DS_Store $(APP_DIST_DIR)/Applications -$(APP_DIST_DIR)/Applications: - @rm -f $@ - @cd $(@D); $(LN_S) /Applications $(@F) - -$(APP_DIST_EXTRAS): $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt - -$(OSX_TEMP_ISO): $(APP_DIST_EXTRAS) +$(OSX_DMG): deploydir $(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ $(APP_DIST_DIR) -- $(if $(SOURCE_DATE_EPOCH),-volume_date all_file_dates =$(SOURCE_DATE_EPOCH)) -$(OSX_DMG): $(OSX_TEMP_ISO) - $(DMG) dmg "$<" "$@" - -$(APP_DIST_DIR)/.background/background.tiff: - $(MKDIR_P) $(@D) - cp $(OSX_BACKGROUND_IMAGE) $@ - $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt: $(OSX_APP_BUILT) $(OSX_PACKAGING) - INSTALLNAMETOOL=$(INSTALLNAMETOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) + INSTALL_NAME_TOOL=$(INSTALL_NAME_TOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) -deploydir: $(APP_DIST_EXTRAS) +deploydir: $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt endif !BUILD_DARWIN -appbundle: $(OSX_APP_BUILT) deploy: $(OSX_DMG) endif @@ -6,18 +6,14 @@ https://bitcoincore.org For an immediately usable, binary version of the Bitcoin Core software, see https://bitcoincore.org/en/download/. -Further information about Bitcoin Core is available in the [doc folder](/doc). - -What is Bitcoin? ----------------- +What is Bitcoin Core? +--------------------- -Bitcoin is an experimental digital currency that enables instant payments to -anyone, anywhere in the world. Bitcoin uses peer-to-peer technology to operate -with no central authority: managing transactions and issuing money are carried -out collectively by the network. Bitcoin Core is the name of open source -software which enables the use of this currency. +Bitcoin Core connects to the Bitcoin peer-to-peer network to download and fully +validate blocks and transactions. It also includes a wallet and graphical user +interface, which can be optionally built. -For more information read the original Bitcoin whitepaper. +Further information about Bitcoin Core is available in the [doc folder](/doc). License ------- @@ -112,7 +112,7 @@ /src/dbwrapper.* @jamesob # Linter -/test/lint/lint-shell.sh @hebasto +/test/lint/lint-shell.py @hebasto # Bech32 /src/bech32.* @sipa diff --git a/build-aux/m4/ax_cxx_compile_stdcxx.m4 b/build-aux/m4/ax_cxx_compile_stdcxx.m4 index 43087b2e68..51a35054d0 100644 --- a/build-aux/m4/ax_cxx_compile_stdcxx.m4 +++ b/build-aux/m4/ax_cxx_compile_stdcxx.m4 @@ -10,13 +10,13 @@ # # Check for baseline language coverage in the compiler for the specified # version of the C++ standard. If necessary, add switches to CXX and -# CXXCPP to enable support. VERSION may be '11' (for the C++11 standard) -# or '14' (for the C++14 standard). +# CXXCPP to enable support. VERSION may be '11', '14', '17', or '20' for +# the respective C++ standard version. # # The second argument, if specified, indicates whether you insist on an # extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g. # -std=c++11). If neither is specified, you get whatever works, with -# preference for an extended mode. +# preference for no added switch, and then for an extended mode. # # The third argument, if specified 'mandatory' or if left unspecified, # indicates that baseline support for the specified C++ standard is @@ -35,13 +35,15 @@ # Copyright (c) 2015 Moritz Klammler <moritz@klammler.eu> # Copyright (c) 2016, 2018 Krzesimir Nowak <qdlacz@gmail.com> # Copyright (c) 2019 Enji Cooper <yaneurabeya@gmail.com> +# Copyright (c) 2020 Jason Merrill <jason@redhat.com> +# Copyright (c) 2021 Jörn Heusipp <osmanx@problemloesungsmaschine.de> # # Copying and distribution of this file, with or without modification, are # permitted in any medium without royalty provided the copyright notice # and this notice are preserved. This file is offered as-is, without any # warranty. -#serial 11 +#serial 14 dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro dnl (serial version number 13). @@ -50,6 +52,7 @@ AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"], [$1], [14], [ax_cxx_compile_alternatives="14 1y"], [$1], [17], [ax_cxx_compile_alternatives="17 1z"], + [$1], [20], [ax_cxx_compile_alternatives="20"], [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl m4_if([$2], [], [], [$2], [ext], [], @@ -62,6 +65,16 @@ AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl AC_LANG_PUSH([C++])dnl ac_success=no + m4_if([$2], [], [dnl + AC_CACHE_CHECK(whether $CXX supports C++$1 features by default, + ax_cv_cxx_compile_cxx$1, + [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [ax_cv_cxx_compile_cxx$1=yes], + [ax_cv_cxx_compile_cxx$1=no])]) + if test x$ax_cv_cxx_compile_cxx$1 = xyes; then + ac_success=yes + fi]) + m4_if([$2], [noext], [], [dnl if test x$ac_success = xno; then for alternative in ${ax_cxx_compile_alternatives}; do @@ -140,7 +153,6 @@ m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11], _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 ) - dnl Test body for checking C++14 support m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], @@ -148,12 +160,24 @@ m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 ) +dnl Test body for checking C++17 support + m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17], _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 _AX_CXX_COMPILE_STDCXX_testbody_new_in_17 ) +dnl Test body for checking C++20 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_20], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_17 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_20 +) + + dnl Tests for new features in C++11 m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[ @@ -949,3 +973,33 @@ namespace cxx17 #endif // __cplusplus < 201703L ]]) + + +dnl Tests for new features in C++20 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_20], [[ + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 202002L + +#error "This is not a C++20 compiler" + +#else + +#include <version> + +namespace cxx20 +{ + +// As C++20 supports feature test macros in the standard, there is no +// immediate need to actually test for feature availability on the +// Autoconf side. + +} // namespace cxx20 + +#endif // __cplusplus < 202002L + +]]) diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index 1454e33f6e..a716cd9a27 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -116,8 +116,8 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ BITCOIN_QT_CHECK([ TEMP_CPPFLAGS=$CPPFLAGS TEMP_CXXFLAGS=$CXXFLAGS - CPPFLAGS="$QT_INCLUDES $CPPFLAGS" - CXXFLAGS="$PIC_FLAGS $CXXFLAGS" + CPPFLAGS="$QT_INCLUDES $CORE_CPPFLAGS $CPPFLAGS" + CXXFLAGS="$PIC_FLAGS $CORE_CXXFLAGS $CXXFLAGS" _BITCOIN_QT_IS_STATIC if test "$bitcoin_cv_static_qt" = "yes"; then _BITCOIN_QT_CHECK_STATIC_LIBS @@ -178,8 +178,8 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ AC_MSG_CHECKING([whether -fPIE can be used with this Qt config]) TEMP_CPPFLAGS=$CPPFLAGS TEMP_CXXFLAGS=$CXXFLAGS - CPPFLAGS="$QT_INCLUDES $CPPFLAGS" - CXXFLAGS="$PIE_FLAGS $CXXFLAGS" + CPPFLAGS="$QT_INCLUDES $CORE_CPPFLAGS $CPPFLAGS" + CXXFLAGS="$PIE_FLAGS $CORE_CXXFLAGS $CXXFLAGS" AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include <QtCore/qconfig.h> #ifndef QT_VERSION @@ -201,7 +201,7 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ BITCOIN_QT_CHECK([ AC_MSG_CHECKING([whether -fPIC is needed with this Qt config]) TEMP_CPPFLAGS=$CPPFLAGS - CPPFLAGS="$QT_INCLUDES $CPPFLAGS" + CPPFLAGS="$QT_INCLUDES $CORE_CPPFLAGS $CPPFLAGS" AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include <QtCore/qconfig.h> #ifndef QT_VERSION diff --git a/build_msvc/.gitignore b/build_msvc/.gitignore index b0e557bc0c..b2eb9313a0 100644 --- a/build_msvc/.gitignore +++ b/build_msvc/.gitignore @@ -22,6 +22,7 @@ bench_bitcoin/bench_bitcoin.vcxproj libtest_util/libtest_util.vcxproj /bitcoin_config.h +/common.init.vcxproj */Win32 libbitcoin_qt/QtGeneratedFiles/* diff --git a/build_msvc/README.md b/build_msvc/README.md index cabe4d55ec..7feee6b766 100644 --- a/build_msvc/README.md +++ b/build_msvc/README.md @@ -28,7 +28,7 @@ Qt --------------------- To build Bitcoin Core with the GUI, a static build of Qt is required. -1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-src-5.15.2.zip`](https://download.qt.io/official_releases/qt/5.15/5.15.2/single/qt-everywhere-src-5.15.2.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`. +1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-opensource-src-5.15.3.zip`](https://download.qt.io/official_releases/qt/5.15/5.15.3/single/qt-everywhere-opensource-src-5.15.3.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`. 2. Open "x64 Native Tools Command Prompt for VS 2019", and input the following commands: ```cmd @@ -83,4 +83,4 @@ If is it enabled then in the output `Dynamic base` will be listed in the `DLL ch Terminal Server Aware ``` -This may not disable all stack randomization as versions of windows employ additional stack randomization protections. These protections must be turned off in the OS configuration.
\ No newline at end of file +This may not disable all stack randomization as versions of windows employ additional stack randomization protections. These protections must be turned off in the OS configuration. diff --git a/build_msvc/bitcoin_config.h.in b/build_msvc/bitcoin_config.h.in index e25024e871..b37d536947 100644 --- a/build_msvc/bitcoin_config.h.in +++ b/build_msvc/bitcoin_config.h.in @@ -125,10 +125,6 @@ don't. */ #define HAVE_DECL_STRERROR_R 0 -/* Define to 1 if you have the declaration of `strnlen', and to 0 if you - don't. */ -#define HAVE_DECL_STRNLEN 1 - /* Define if the dllexport attribute is supported. */ #define HAVE_DLLEXPORT_ATTRIBUTE 1 diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj.in index 0cbe2effd5..182efff233 100644 --- a/build_msvc/common.init.vcxproj +++ b/build_msvc/common.init.vcxproj.in @@ -39,7 +39,7 @@ <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration"> <LinkIncremental>false</LinkIncremental> <UseDebugLibraries>false</UseDebugLibraries> - <PlatformToolset>v142</PlatformToolset> + <PlatformToolset>@TOOLSET@</PlatformToolset> <CharacterSet>Unicode</CharacterSet> <GenerateManifest>No</GenerateManifest> <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</OutDir> @@ -49,7 +49,7 @@ <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration"> <LinkIncremental>true</LinkIncremental> <UseDebugLibraries>true</UseDebugLibraries> - <PlatformToolset>v142</PlatformToolset> + <PlatformToolset>@TOOLSET@</PlatformToolset> <CharacterSet>Unicode</CharacterSet> <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</OutDir> <IntDir>$(Platform)\$(Configuration)\$(ProjectName)\</IntDir> diff --git a/build_msvc/libsecp256k1/libsecp256k1.vcxproj b/build_msvc/libsecp256k1/libsecp256k1.vcxproj index f9b0a7975c..16ee32d87e 100644 --- a/build_msvc/libsecp256k1/libsecp256k1.vcxproj +++ b/build_msvc/libsecp256k1/libsecp256k1.vcxproj @@ -8,6 +8,8 @@ <ConfigurationType>StaticLibrary</ConfigurationType> </PropertyGroup> <ItemGroup> + <ClCompile Include="..\..\src\secp256k1\src\precomputed_ecmult.c" /> + <ClCompile Include="..\..\src\secp256k1\src\precomputed_ecmult_gen.c" /> <ClCompile Include="..\..\src\secp256k1\src\secp256k1.c" /> </ItemGroup> <ItemDefinitionGroup> diff --git a/build_msvc/msvc-autogen.py b/build_msvc/msvc-autogen.py index 2a70cd9332..819fe1b7ae 100755 --- a/build_msvc/msvc-autogen.py +++ b/build_msvc/msvc-autogen.py @@ -50,13 +50,6 @@ def parse_makefile(makefile): lib_sources[current_lib] = [] break -def set_common_properties(toolset): - with open(os.path.join(SOURCE_DIR, '../build_msvc/common.init.vcxproj'), 'r', encoding='utf-8') as rfile: - s = rfile.read() - s = re.sub('<PlatformToolset>.*?</PlatformToolset>', '<PlatformToolset>'+toolset+'</PlatformToolset>', s) - with open(os.path.join(SOURCE_DIR, '../build_msvc/common.init.vcxproj'), 'w', encoding='utf-8',newline='\n') as wfile: - wfile.write(s) - def parse_config_into_btc_config(): def find_between( s, first, last ): try: @@ -92,13 +85,18 @@ def parse_config_into_btc_config(): with open(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h'), "w", encoding="utf8") as btc_config: btc_config.writelines(template) +def set_properties(vcxproj_filename, placeholder, content): + with open(vcxproj_filename + '.in', 'r', encoding='utf-8') as vcxproj_in_file: + with open(vcxproj_filename, 'w', encoding='utf-8') as vcxproj_file: + vcxproj_file.write(vcxproj_in_file.read().replace(placeholder, content)) + def main(): parser = argparse.ArgumentParser(description='Bitcoin-core msbuild configuration initialiser.') - parser.add_argument('-toolset', nargs='?',help='Optionally sets the msbuild platform toolset, e.g. v142 for Visual Studio 2019.' + parser.add_argument('-toolset', nargs='?', default=DEFAULT_PLATFORM_TOOLSET, + help='Optionally sets the msbuild platform toolset, e.g. v142 for Visual Studio 2019.' ' default is %s.'%DEFAULT_PLATFORM_TOOLSET) args = parser.parse_args() - if args.toolset: - set_common_properties(args.toolset) + set_properties(os.path.join(SOURCE_DIR, '../build_msvc/common.init.vcxproj'), '@TOOLSET@', args.toolset) for makefile_name in os.listdir(SOURCE_DIR): if 'Makefile' in makefile_name: @@ -110,10 +108,7 @@ def main(): content += ' <ClCompile Include="..\\..\\src\\' + source_filename + '">\n' content += ' <ObjectFileName>$(IntDir)' + object_filename + '</ObjectFileName>\n' content += ' </ClCompile>\n' - with open(vcxproj_filename + '.in', 'r', encoding='utf-8') as vcxproj_in_file: - with open(vcxproj_filename, 'w', encoding='utf-8') as vcxproj_file: - vcxproj_file.write(vcxproj_in_file.read().replace( - '@SOURCE_FILES@\n', content)) + set_properties(vcxproj_filename, '@SOURCE_FILES@\n', content) parse_config_into_btc_config() copyfile(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h'), os.path.join(SOURCE_DIR, 'config/bitcoin-config.h')) copyfile(os.path.join(SOURCE_DIR,'../build_msvc/libsecp256k1_config.h'), os.path.join(SOURCE_DIR, 'secp256k1/src/libsecp256k1-config.h')) diff --git a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj index d3be693e99..ec572b4f2e 100644 --- a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj +++ b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj @@ -12,6 +12,7 @@ <ClCompile Include="..\..\src\test\util\setup_common.cpp" /> <ClCompile Include="..\..\src\qt\test\addressbooktests.cpp" /> <ClCompile Include="..\..\src\qt\test\apptests.cpp" /> + <ClCompile Include="..\..\src\qt\test\optiontests.cpp" /> <ClCompile Include="..\..\src\qt\test\rpcnestedtests.cpp" /> <ClCompile Include="..\..\src\qt\test\test_main.cpp" /> <ClCompile Include="..\..\src\qt\test\uritests.cpp" /> @@ -20,6 +21,7 @@ <ClCompile Include="..\..\src\wallet\test\wallet_test_fixture.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_addressbooktests.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_apptests.cpp" /> + <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_optiontests.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_rpcnestedtests.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_uritests.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_wallettests.cpp" /> @@ -88,6 +90,7 @@ <ItemGroup> <MocTestFiles Include="..\..\src\qt\test\addressbooktests.h" /> <MocTestFiles Include="..\..\src\qt\test\apptests.h" /> + <MocTestFiles Include="..\..\src\qt\test\optiontests.h" /> <MocTestFiles Include="..\..\src\qt\test\rpcnestedtests.h" /> <MocTestFiles Include="..\..\src\qt\test\uritests.h" /> <MocTestFiles Include="..\..\src\qt\test\wallettests.h" /> diff --git a/build_msvc/vcpkg.json b/build_msvc/vcpkg.json index d8753ec21f..86773d1fd3 100644 --- a/build_msvc/vcpkg.json +++ b/build_msvc/vcpkg.json @@ -8,7 +8,6 @@ "boost-signals2", "boost-test", "sqlite3", - "double-conversion", { "name": "libevent", "features": ["thread"] diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index 1518c033f4..8330df87eb 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -11,9 +11,9 @@ ${CI_RETRY_EXE} apt-get install -y clang-format-9 python3-pip curl git gawk jq update-alternatives --install /usr/bin/clang-format clang-format "$(which clang-format-9 )" 100 update-alternatives --install /usr/bin/clang-format-diff clang-format-diff "$(which clang-format-diff-9)" 100 -${CI_RETRY_EXE} pip3 install codespell==2.0.0 -${CI_RETRY_EXE} pip3 install flake8==3.8.3 -${CI_RETRY_EXE} pip3 install mypy==0.910 +${CI_RETRY_EXE} pip3 install codespell==2.1.0 +${CI_RETRY_EXE} pip3 install flake8==4.0.1 +${CI_RETRY_EXE} pip3 install mypy==0.942 ${CI_RETRY_EXE} pip3 install pyzmq==22.3.0 ${CI_RETRY_EXE} pip3 install vulture==2.3 diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index fa9bf4c646..f174b4d074 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -22,7 +22,7 @@ test/lint/git-subtree-check.sh src/univalue test/lint/git-subtree-check.sh src/leveldb test/lint/git-subtree-check.sh src/crc32c test/lint/check-doc.py -test/lint/lint-all.sh +test/lint/lint-all.py if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ "$CIRRUS_PR" = "" ] ; then # Sanity check only the last few commits to get notified of missing sigs, diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index e806683128..5a150d5f80 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -37,6 +37,7 @@ export USE_BUSY_BOX=${USE_BUSY_BOX:-false} export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true} +export RUN_TIDY=${RUN_TIDY:-false} export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false} # By how much to scale the test_runner timeouts (option --timeout-factor). # This is needed because some ci machines have slow CPU or disk, so sanitizers diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh index b333635759..766424769d 100755 --- a/ci/test/00_setup_env_i686_multiprocess.sh +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -15,4 +15,3 @@ export GOAL="install" export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' LDFLAGS='--rtlib=compiler-rt -lgcc_s'" export TEST_RUNNER_ENV="BITCOIND=bitcoin-node" export TEST_RUNNER_EXTRA="--nosandbox" -export PIP_PACKAGES="lief" diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index b03a7edb54..69883e3609 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -11,4 +11,4 @@ export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent- export DOCKER_NAME_TAG=ubuntu:22.04 export NO_DEPENDS=1 export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" +export BITCOIN_CONFIG="--enable-c++20 --enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 619acc4677..071bac8fb3 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -13,10 +13,11 @@ LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_ export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_msan" -export PACKAGES="clang-9 llvm-9 cmake" +export PACKAGES="clang-12 llvm-12 cmake" +# BDB generates false-positives and will be removed in future export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" export RUN_UNIT_TESTS="false" export RUN_FUNCTIONAL_TESTS="false" diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index a7715a6ded..9477fb2d9f 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -15,5 +15,6 @@ export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export FUZZ_TESTS_CONFIG="--valgrind" export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++" +# Temporarily pin dwarf 4, until valgrind can understand clang's dwarf 5 +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++ CXXFLAGS='-fdebug-default-version=4'" export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index b8418106c6..34a792ec8f 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -11,13 +11,13 @@ LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" -export BDB_PREFIX="${BASE_ROOT_DIR}/db4" export CONTAINER_NAME="ci_native_msan" -export PACKAGES="clang-9 llvm-9 cmake" +export PACKAGES="clang-12 llvm-12 cmake" +# BDB generates false-positives and will be removed in future export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--enable-wallet --with-sanitizers=memory --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' BDB_LIBS='-L${BDB_PREFIX}/lib -ldb_cxx-4.8' BDB_CFLAGS='-I${BDB_PREFIX}/include'" +export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" export RUN_FUNCTIONAL_TESTS="false" export CCACHE_SIZE=250M diff --git a/ci/test/00_setup_env_native_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh deleted file mode 100755 index d80a7f9633..0000000000 --- a/ci/test/00_setup_env_native_nowallet.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -export CONTAINER_NAME=ci_native_nowallet -export DOCKER_NAME_TAG=ubuntu:18.04 # Use bionic to have one config run the tests in python3.6, see doc/dependencies.md -export PACKAGES="python3-zmq clang-7 llvm-7 libc++abi-7-dev libc++-7-dev" # Use clang-7 to test C++17 compatibility, see doc/dependencies.md -export DEP_OPTS="NO_WALLET=1 CC=clang-7 CXX='clang++-7 -stdlib=libc++'" -export GOAL="install" -export BITCOIN_CONFIG="--enable-reduce-exports CC=clang-7 CXX='clang++-7 -stdlib=libc++'" diff --git a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh new file mode 100755 index 0000000000..63560a5f5c --- /dev/null +++ b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel +export DOCKER_NAME_TAG=ubuntu:18.04 # Use bionic to have one config run the tests in python3.6, see doc/dependencies.md +export PACKAGES="python3-zmq clang-8 llvm-8 libc++abi-8-dev libc++-8-dev" # Use clang-8 to test C++17 compatibility, see doc/dependencies.md +export DEP_OPTS="NO_WALLET=1 CC=clang-8 CXX='clang++-8 -stdlib=libc++'" +export GOAL="install" +export BITCOIN_CONFIG="--enable-reduce-exports CC=clang-8 CXX='clang++-8 -stdlib=libc++' --enable-experimental-util-chainstate --with-experimental-kernel-lib --enable-shared" diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index f8b366e77f..f399e43612 100755 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -7,13 +7,13 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_qt5 -export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic gcc-8 can compile our C++17 and run our functional tests in python3, see doc/dependencies.md +export DOCKER_NAME_TAG=debian:buster # Check that buster gcc-8 can compile our C++17 and run our functional tests in python3, see doc/dependencies.md export PACKAGES="gcc-8 g++-8 python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" export DEP_OPTS="NO_QT=1 NO_UPNP=1 NO_NATPMP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1 CC=gcc-8 CXX=g++-8" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" -export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1 v0.20.1 v0.21.0 v22.0" +export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.14.3 v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1 v0.20.1 v0.21.0 v22.0 v23.0" export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-reduce-exports \ ---enable-debug --disable-fuzz-binary CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" CC=gcc-8 CXX=g++-8" +--enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" CC=gcc-8 CXX=g++-8" diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh new file mode 100755 index 0000000000..e4d3468473 --- /dev/null +++ b/ci/test/00_setup_env_native_tidy.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export DOCKER_NAME_TAG="ubuntu:22.04" +export CONTAINER_NAME=ci_native_tidy +export PACKAGES="clang libclang-dev llvm-dev clang-tidy bear cmake libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" +export NO_DEPENDS=1 +export RUN_UNIT_TESTS=false +export RUN_FUNCTIONAL_TESTS=false +export RUN_FUZZ_TESTS=false +export RUN_TIDY=true +export GOAL="install" +export BITCOIN_CONFIG="CC=clang CXX=clang++ --with-incompatible-bdb --disable-hardening CFLAGS='-O0 -g0' CXXFLAGS='-O0 -g0'" +export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 0036255caf..ae942d892b 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan export DOCKER_NAME_TAG=ubuntu:22.04 -export PACKAGES="clang llvm libc++abi-dev libc++-dev python3-zmq" -export DEP_OPTS="CC=clang CXX='clang++ -stdlib=libc++'" +export PACKAGES="clang-13 llvm-13 libc++abi-13-dev libc++-13-dev python3-zmq" +export DEP_OPTS="CC=clang-13 CXX='clang++-13 -stdlib=libc++'" export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' CXXFLAGS='-g' --with-sanitizers=thread CC=clang CXX='clang++ -stdlib=libc++'" +export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' CXXFLAGS='-g' --with-sanitizers=thread CC=clang-13 CXX='clang++-13 -stdlib=libc++'" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 646070a84e..7b714dff5c 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -13,4 +13,5 @@ export USE_VALGRIND=1 export NO_DEPENDS=1 export TEST_RUNNER_EXTRA="--nosandbox --exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++" # TODO enable GUI +# Temporarily pin dwarf 4, until valgrind can understand clang's dwarf 5 +export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++ CXXFLAGS='-fdebug-default-version=4'" # TODO enable GUI diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index e409df62eb..453a34ca78 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -103,14 +103,24 @@ fi CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - CI_EXEC "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-9) 100" - CI_EXEC "update-alternatives --install /usr/bin/clang clang \$(which clang-9) 100" + CI_EXEC "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-12) 100" + CI_EXEC "update-alternatives --install /usr/bin/clang clang \$(which clang-12) 100" CI_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/" CI_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-12.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project" - CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=Memory -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/" + CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/" CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx" fi +if [[ "${RUN_TIDY}" == "true" ]]; then + export DIR_IWYU="${BASE_SCRATCH_DIR}/iwyu" + if [ ! -d "${DIR_IWYU}" ]; then + CI_EXEC "mkdir -p ${DIR_IWYU}/build/" + CI_EXEC "git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_14 ${DIR_IWYU}/include-what-you-use" + CI_EXEC "cd ${DIR_IWYU}/build && cmake -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-14 ../include-what-you-use" + CI_EXEC "cd ${DIR_IWYU}/build && make install $MAKEJOBS" + fi +fi + if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then echo "Create $BASE_ROOT_DIR" CI_EXEC rsync -a /ro_base/ "$BASE_ROOT_DIR" diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index 8f75fbd1fa..fc2f76797c 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -36,13 +36,6 @@ if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then CI_EXEC "yes | ${ANDROID_HOME}/cmdline-tools/tools/bin/sdkmanager --install \"build-tools;${ANDROID_BUILD_TOOLS_VERSION}\" \"platform-tools\" \"platforms;android-${ANDROID_API_LEVEL}\" \"ndk;${ANDROID_NDK_VERSION}\"" fi -if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - # Use BDB compiled using install_db4.sh script to work around linking issue when using BDB - # from depends. See https://github.com/bitcoin/bitcoin/pull/18288#discussion_r433189350 for - # details. - CI_EXEC "contrib/install_db4.sh \$(pwd) --enable-umrw CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" -fi - if [ -z "$NO_DEPENDS" ]; then if [[ $DOCKER_NAME_TAG == *centos* ]]; then # CentOS has problems building the depends if the config shell is not explicitly set @@ -52,7 +45,7 @@ if [ -z "$NO_DEPENDS" ]; then else SHELL_OPTS="CONFIG_SHELL=" fi - CI_EXEC "$SHELL_OPTS" make "$MAKEJOBS" -C depends HOST="$HOST" "$DEP_OPTS" + CI_EXEC "$SHELL_OPTS" make "$MAKEJOBS" -C depends HOST="$HOST" "$DEP_OPTS" LOG=1 fi if [ -n "$PREVIOUS_RELEASES_TO_DOWNLOAD" ]; then CI_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}" diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index 7158b69b4e..6a6bde05a1 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -48,7 +48,12 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then CI_EXEC 'grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h' fi -CI_EXEC make "$MAKEJOBS" "$GOAL" || ( echo "Build failure. Verbose build follows." && CI_EXEC make "$GOAL" V=1 ; false ) +if [[ "${RUN_TIDY}" == "true" ]]; then + MAYBE_BEAR="bear --config src/.bear-tidy-config" + MAYBE_TOKEN="--" +fi + +CI_EXEC "${MAYBE_BEAR}" "${MAYBE_TOKEN}" make "$MAKEJOBS" "$GOAL" || ( echo "Build failure. Verbose build follows." && CI_EXEC make "$GOAL" V=1 ; false ) CI_EXEC "ccache --version | head -n 1 && ccache --show-stats" CI_EXEC du -sh "${DEPENDS_DIR}"/*/ diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index de42aa6eb1..e64af2ad5d 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -27,13 +27,25 @@ if [ "$RUN_UNIT_TESTS" = "true" ]; then fi if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then - CI_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${BASE_BUILD_DIR}/bitcoin-*/src/test/test_bitcoin*" --catch_system_errors=no -l test_suite + CI_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${BASE_OUTDIR}/bin/test_bitcoin" --catch_system_errors=no -l test_suite fi if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${TEST_RUNNER_ENV}" test/functional/test_runner.py --ci "$MAKEJOBS" --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" "${TEST_RUNNER_EXTRA}" --quiet --failfast fi +if [ "${RUN_TIDY}" = "true" ]; then + export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST/src/" + CI_EXEC run-clang-tidy "${MAKEJOBS}" + export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST/" + CI_EXEC "python3 ${DIR_IWYU}/include-what-you-use/iwyu_tool.py"\ + " src/compat"\ + " src/init"\ + " src/rpc/fees.cpp"\ + " src/rpc/signmessage.cpp"\ + " -p . ${MAKEJOBS} -- -Xiwyu --cxx17ns -Xiwyu --mapping_file=${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp" +fi + if [ "$RUN_SECURITY_TESTS" = "true" ]; then CI_EXEC make test-security-check fi diff --git a/ci/test/wrapped-cl.bat b/ci/test/wrapped-cl.bat new file mode 100644 index 0000000000..fc2a604c58 --- /dev/null +++ b/ci/test/wrapped-cl.bat @@ -0,0 +1 @@ +ccache cl %* diff --git a/configure.ac b/configure.ac index 22dc2e01a6..6dcc5183ea 100644 --- a/configure.ac +++ b/configure.ac @@ -1,5 +1,5 @@ AC_PREREQ([2.69]) -define(_CLIENT_VERSION_MAJOR, 22) +define(_CLIENT_VERSION_MAJOR, 23) define(_CLIENT_VERSION_MINOR, 99) define(_CLIENT_VERSION_BUILD, 0) define(_CLIENT_VERSION_RC, 0) @@ -24,6 +24,7 @@ BITCOIN_GUI_NAME=bitcoin-qt BITCOIN_CLI_NAME=bitcoin-cli BITCOIN_TX_NAME=bitcoin-tx BITCOIN_UTIL_NAME=bitcoin-util +BITCOIN_CHAINSTATE_NAME=bitcoin-chainstate BITCOIN_WALLET_TOOL_NAME=bitcoin-wallet dnl Multi Process BITCOIN_MP_NODE_NAME=bitcoin-node @@ -41,14 +42,9 @@ AH_TOP([#ifndef BITCOIN_CONFIG_H]) AH_TOP([#define BITCOIN_CONFIG_H]) AH_BOTTOM([#endif //BITCOIN_CONFIG_H]) -dnl faketime breaks configure and is only needed for make. Disable it here. -unset FAKETIME - dnl Automake init set-up and checks AM_INIT_AUTOMAKE([1.13 no-define subdir-objects foreign]) -dnl faketime messes with timestamps and causes configure to be re-run. -dnl --disable-maintainer-mode can be used to bypass this. AM_MAINTAINER_MODE([enable]) dnl make the compilation flags quiet unless V=1 is used @@ -77,8 +73,18 @@ AC_ARG_WITH([seccomp], [seccomp_found=$withval], [seccomp_found=auto]) +AC_ARG_ENABLE([c++20], + [AS_HELP_STRING([--enable-c++20], + [enable compilation in c++20 mode (disabled by default)])], + [use_cxx20=$enableval], + [use_cxx20=no]) + dnl Require C++17 compiler (no GNU extensions) +if test "$use_cxx20" = "no"; then AX_CXX_COMPILE_STDCXX([17], [noext], [mandatory]) +else +AX_CXX_COMPILE_STDCXX([20], [noext], [mandatory]) +fi dnl Check if -latomic is required for <std::atomic> CHECK_ATOMIC @@ -95,17 +101,13 @@ fi AC_PROG_OBJCXX ]) -dnl Since libtool 1.5.2 (released 2004-01-25), on Linux libtool no longer -dnl sets RPATH for any directories in the dynamic linker search path. -dnl See more: https://wiki.debian.org/RpathIssue -LT_PREREQ([1.5.2]) +dnl OpenBSD ships with 2.4.2 +LT_PREREQ([2.4.2]) dnl Libtool init checks. LT_INIT([pic-only win32-dll]) dnl Check/return PATH for base programs. AC_PATH_TOOL([AR], [ar]) -AC_PATH_TOOL([RANLIB], [ranlib]) -AC_PATH_TOOL([STRIP], [strip]) AC_PATH_TOOL([GCOV], [gcov]) AC_PATH_TOOL([LLVM_COV], [llvm-cov]) AC_PATH_PROG([LCOV], [lcov]) @@ -357,7 +359,9 @@ case $host in esac if test "$enable_debug" = "yes"; then - dnl Clear default -g -O2 flags + dnl If debugging is enabled, and the user hasn't overridden CXXFLAGS, clear + dnl them, to prevent autoconfs "-g -O2" being added. Otherwise we'd end up + dnl with "-O0 -g3 -g -O2". if test "$CXXFLAGS_overridden" = "no"; then CXXFLAGS="" fi @@ -464,7 +468,7 @@ if test "$CXXFLAGS_overridden" = "no"; then fi dnl Don't allow extended (non-ASCII) symbols in identifiers. This is easier for code review. -AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers], [CXXFLAGS="$CXXFLAGS -fno-extended-identifiers"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers], [CORE_CXXFLAGS="$CORE_CXXFLAGS -fno-extended-identifiers"], [], [$CXXFLAG_WERROR]) enable_arm_crc=no enable_arm_shani=no @@ -615,7 +619,7 @@ CXXFLAGS="$TEMP_CXXFLAGS" fi -CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO" +CORE_CPPFLAGS="$CORE_CPPFLAGS -DHAVE_BUILD_INFO" AC_ARG_WITH([utils], [AS_HELP_STRING([--with-utils], @@ -647,12 +651,24 @@ AC_ARG_ENABLE([util-util], [build_bitcoin_util=$enableval], [build_bitcoin_util=$build_bitcoin_utils]) +AC_ARG_ENABLE([experimental-util-chainstate], + [AS_HELP_STRING([--enable-experimental-util-chainstate], + [build experimental bitcoin-chainstate executable (default=no)])], + [build_bitcoin_chainstate=$enableval], + [build_bitcoin_chainstate=no]) + AC_ARG_WITH([libs], [AS_HELP_STRING([--with-libs], [build libraries (default=yes)])], [build_bitcoin_libs=$withval], [build_bitcoin_libs=yes]) +AC_ARG_WITH([experimental-kernel-lib], + [AS_HELP_STRING([--with-experimental-kernel-lib], + [build experimental bitcoinkernel library (default is to build if we're building libraries and the experimental build-chainstate executable)])], + [build_experimental_kernel_lib=$withval], + [build_experimental_kernel_lib=auto]) + AC_ARG_WITH([daemon], [AS_HELP_STRING([--with-daemon], [build bitcoind daemon (default=yes)])], @@ -691,7 +707,7 @@ case $host in AC_MSG_ERROR([windres not found]) fi - CPPFLAGS="$CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -D_WIN32_WINNT=0x0601 -D_WIN32_IE=0x0501 -DWIN32_LEAN_AND_MEAN" + CORE_CPPFLAGS="$CORE_CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -D_WIN32_WINNT=0x0601 -D_WIN32_IE=0x0501 -DWIN32_LEAN_AND_MEAN" dnl libtool insists upon adding -nostdlib and a list of objects/libs to link against. dnl That breaks our ability to build dll's with static libgcc/libstdc++/libssp. Override @@ -702,7 +718,7 @@ case $host in postdeps_CXX= dnl We require Windows 7 (NT 6.1) or later - AX_CHECK_LINK_FLAG([-Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1], [LDFLAGS="$LDFLAGS -Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1"], [], [$LDFLAG_WERROR]) ;; *darwin*) TARGET_OS=darwin @@ -740,20 +756,20 @@ case $host in if test "$use_upnp" != "no" && $BREW list --versions miniupnpc >/dev/null; then miniupnpc_prefix=$($BREW --prefix miniupnpc 2>/dev/null) if test "$suppress_external_warnings" != "no"; then - CPPFLAGS="$CPPFLAGS -isystem $miniupnpc_prefix/include" + MINIUPNPC_CPPFLAGS="-isystem $miniupnpc_prefix/include" else - CPPFLAGS="$CPPFLAGS -I$miniupnpc_prefix/include" + MINIUPNPC_CPPFLAGS="-I$miniupnpc_prefix/include" fi - LDFLAGS="$LDFLAGS -L$miniupnpc_prefix/lib" + MINIUPNPC_LIBS="-L$miniupnpc_prefix/lib" fi if test "$use_natpmp" != "no" && $BREW list --versions libnatpmp >/dev/null; then libnatpmp_prefix=$($BREW --prefix libnatpmp 2>/dev/null) if test "$suppress_external_warnings" != "no"; then - CPPFLAGS="$CPPFLAGS -isystem $libnatpmp_prefix/include" + NATPMP_CPPFLAGS="-isystem $libnatpmp_prefix/include" else - CPPFLAGS="$CPPFLAGS -I$libnatpmp_prefix/include" + NATPMP_CPPFLAGS="-I$libnatpmp_prefix/include" fi - LDFLAGS="$LDFLAGS -L$libnatpmp_prefix/lib" + NATPMP_LIBS="-L$libnatpmp_prefix/lib" fi ;; esac @@ -765,10 +781,9 @@ case $host in ;; *) AC_PATH_TOOL([DSYMUTIL], [dsymutil], [dsymutil]) - AC_PATH_TOOL([INSTALLNAMETOOL], [install_name_tool], [install_name_tool]) + AC_PATH_TOOL([INSTALL_NAME_TOOL], [install_name_tool], [install_name_tool]) AC_PATH_TOOL([OTOOL], [otool], [otool]) AC_PATH_PROGS([XORRISOFS], [xorrisofs], [xorrisofs]) - AC_PATH_PROGS([DMG], [dmg], [dmg]) dnl libtool will try to strip the static lib, which is a problem for dnl cross-builds because strip attempts to call a hard-coded ld, @@ -779,8 +794,8 @@ case $host in esac fi - AX_CHECK_LINK_FLAG([-Wl,-headerpad_max_install_names], [LDFLAGS="$LDFLAGS -Wl,-headerpad_max_install_names"], [], [$LDFLAG_WERROR]) - CPPFLAGS="$CPPFLAGS -DMAC_OSX -DOBJC_OLD_DISPATCH_PROTOTYPES=0" + AX_CHECK_LINK_FLAG([-Wl,-headerpad_max_install_names], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-headerpad_max_install_names"], [], [$LDFLAG_WERROR]) + CORE_CPPFLAGS="$CORE_CPPFLAGS -DMAC_OSX -DOBJC_OLD_DISPATCH_PROTOTYPES=0" OBJCXXFLAGS="$CXXFLAGS" ;; *android*) @@ -843,11 +858,17 @@ if test "$use_lcov" = "yes"; then AC_SUBST(COV_TOOL_WRAPPER, "cov_tool_wrapper.sh") LCOV="$LCOV --gcov-tool $(pwd)/$COV_TOOL_WRAPPER" - AX_CHECK_LINK_FLAG([--coverage], [LDFLAGS="$LDFLAGS --coverage"], + AX_CHECK_LINK_FLAG([--coverage], [CORE_LDFLAGS="$CORE_LDFLAGS --coverage"], [AC_MSG_ERROR([lcov testing requested but --coverage linker flag does not work])]) - AX_CHECK_COMPILE_FLAG([--coverage],[CXXFLAGS="$CXXFLAGS --coverage"], + AX_CHECK_COMPILE_FLAG([--coverage],[CORE_CXXFLAGS="$CORE_CXXFLAGS --coverage"], [AC_MSG_ERROR([lcov testing requested but --coverage flag does not work])]) - CXXFLAGS="$CXXFLAGS -Og" + dnl If coverage is enabled, and the user hasn't overridden CXXFLAGS, clear + dnl them, to prevent autoconfs "-g -O2" being added. Otherwise we'd end up + dnl with "--coverage -Og -O0 -g -O2". + if test "$CXXFLAGS_overridden" = "no"; then + CXXFLAGS="" + fi + CORE_CXXFLAGS="$CORE_CXXFLAGS -Og -O0" fi if test "$use_lcov_branch" != "no"; then @@ -870,13 +891,13 @@ AC_FUNC_STRERROR_R if test "$ac_cv_sys_file_offset_bits" != "" && test "$ac_cv_sys_file_offset_bits" != "no" && test "$ac_cv_sys_file_offset_bits" != "unknown"; then - CPPFLAGS="$CPPFLAGS -D_FILE_OFFSET_BITS=$ac_cv_sys_file_offset_bits" + CORE_CPPFLAGS="$CORE_CPPFLAGS -D_FILE_OFFSET_BITS=$ac_cv_sys_file_offset_bits" fi if test "$ac_cv_sys_large_files" != "" && test "$ac_cv_sys_large_files" != "no" && test "$ac_cv_sys_large_files" != "unknown"; then - CPPFLAGS="$CPPFLAGS -D_LARGE_FILES=$ac_cv_sys_large_files" + CORE_CPPFLAGS="$CORE_CPPFLAGS -D_LARGE_FILES=$ac_cv_sys_large_files" fi AC_SEARCH_LIBS([clock_gettime],[rt]) @@ -960,8 +981,8 @@ dnl These flags are specific to ld64, and may cause issues with other linkers. dnl For example: GNU ld will interpret -dead_strip as -de and then try and use dnl "ad_strip" as the symbol for the entry point. if test "$TARGET_OS" = "darwin"; then - AX_CHECK_LINK_FLAG([-Wl,-dead_strip], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"], [], [$LDFLAG_WERROR]) - AX_CHECK_LINK_FLAG([-Wl,-dead_strip_dylibs], [LDFLAGS="$LDFLAGS -Wl,-dead_strip_dylibs"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-dead_strip], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-dead_strip"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-dead_strip_dylibs], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-dead_strip_dylibs"], [], [$LDFLAG_WERROR]) AX_CHECK_LINK_FLAG([-Wl,-bind_at_load], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-bind_at_load"], [], [$LDFLAG_WERROR]) fi @@ -971,7 +992,6 @@ AC_CHECK_DECLS([getifaddrs, freeifaddrs],[CHECK_SOCKET],, [#include <sys/types.h> #include <ifaddrs.h>] ) -AC_CHECK_DECLS([strnlen]) dnl These are used for daemonization in bitcoind AC_CHECK_DECLS([fork]) @@ -1261,7 +1281,7 @@ dnl Do not change "-I/usr/include" to "-isystem /usr/include" because that dnl is not necessary (/usr/include is already a system directory) and because dnl it would break GCC's #include_next. AC_DEFUN([SUPPRESS_WARNINGS], - [$(echo $1 |${SED} -E -e 's/(^| )-I/\1-isystem /g' -e 's;-isystem /usr/include([/ ]|$);-I/usr/include\1;g')]) + [[$(echo $1 |${SED} -E -e 's/(^| )-I/\1-isystem /g' -e 's;-isystem /usr/include/*( |$);-I/usr/include\1;g')]]) dnl enable-fuzz should disable all other targets if test "$enable_fuzz" = "yes"; then @@ -1270,6 +1290,7 @@ if test "$enable_fuzz" = "yes"; then build_bitcoin_cli=no build_bitcoin_tx=no build_bitcoin_util=no + build_bitcoin_chainstate=no build_bitcoin_wallet=no build_bitcoind=no build_bitcoin_libs=no @@ -1284,26 +1305,11 @@ if test "$enable_fuzz" = "yes"; then enable_fuzz_binary=yes AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"], [], [$CXXFLAG_WERROR]) - - AC_MSG_CHECKING([whether main function is needed for fuzz binary]) - AX_CHECK_LINK_FLAG( - [-fsanitize=$use_sanitizers], - [AC_MSG_RESULT([no])], - [AC_MSG_RESULT([yes]); CPPFLAGS="$CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION"], - [], - [AC_LANG_PROGRAM([[ - #include <cstdint> - #include <cstddef> - extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { return 0; } - /* comment to remove the main function ... - ]],[[ - */ int not_main() { - ]])]) else BITCOIN_QT_INIT dnl sets $bitcoin_enable_qt, $bitcoin_enable_qt_test, $bitcoin_enable_qt_dbus - BITCOIN_QT_CONFIGURE([5.9.5]) + BITCOIN_QT_CONFIGURE([5.11.3]) dnl Keep a copy of the original $QT_INCLUDES and use it when invoking qt's moc QT_INCLUDES_UNSUPPRESSED=$QT_INCLUDES @@ -1312,8 +1318,25 @@ else QT_DBUS_INCLUDES=SUPPRESS_WARNINGS($QT_DBUS_INCLUDES) QT_TEST_INCLUDES=SUPPRESS_WARNINGS($QT_TEST_INCLUDES) fi +fi - CPPFLAGS="$CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION" +if test "$enable_fuzz_binary" = "yes"; then + AC_MSG_CHECKING([whether main function is needed for fuzz binary]) + AX_CHECK_LINK_FLAG( + [], + [AC_MSG_RESULT([no])], + [AC_MSG_RESULT([yes]); CORE_CPPFLAGS="$CORE_CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION"], + [$SANITIZER_LDFLAGS], + [AC_LANG_PROGRAM([[ + #include <cstdint> + #include <cstddef> + extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { return 0; } + /* comment to remove the main function ... + ]],[[ + */ int not_main() { + ]])]) + + CHECK_RUNTIME_LIB fi if test "$enable_wallet" != "no"; then @@ -1365,6 +1388,7 @@ if test "$use_usdt" != "no"; then [AC_MSG_RESULT([no]); use_usdt=no;] ) fi +AM_CONDITIONAL([ENABLE_USDT_TRACEPOINTS], [test "$use_usdt" = "yes"]) if test "$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests" = "nonononononono"; then use_upnp=no @@ -1374,38 +1398,44 @@ fi dnl Check for libminiupnpc (optional) if test "$use_upnp" != "no"; then + TEMP_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $MINIUPNPC_CPPFLAGS" AC_CHECK_HEADERS( [miniupnpc/miniupnpc.h miniupnpc/upnpcommands.h miniupnpc/upnperrors.h], - [AC_CHECK_LIB([miniupnpc], [upnpDiscover], [MINIUPNPC_LIBS=-lminiupnpc], [have_miniupnpc=no])], + [AC_CHECK_LIB([miniupnpc], [upnpDiscover], [MINIUPNPC_LIBS="$MINIUPNPC_LIBS -lminiupnpc"], [have_miniupnpc=no], [$MINIUPNPC_LIBS])], [have_miniupnpc=no] ) -dnl The minimum supported miniUPnPc API version is set to 10. This keeps compatibility -dnl with Ubuntu 16.04 LTS and Debian 8 libminiupnpc-dev packages. -if test "$have_miniupnpc" != "no"; then - AC_MSG_CHECKING([whether miniUPnPc API version is supported]) - AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ - @%:@include <miniupnpc/miniupnpc.h> - ]], [[ - #if MINIUPNPC_API_VERSION >= 10 - // Everything is okay - #else - # error miniUPnPc API version is too old - #endif - ]])],[ - AC_MSG_RESULT([yes]) - ],[ - AC_MSG_RESULT([no]) - AC_MSG_WARN([miniUPnPc API version < 10 is unsupported, disabling UPnP support.]) - have_miniupnpc=no - ]) -fi + dnl The minimum supported miniUPnPc API version is set to 10. This keeps compatibility + dnl with Ubuntu 16.04 LTS and Debian 8 libminiupnpc-dev packages. + if test "$have_miniupnpc" != "no"; then + AC_MSG_CHECKING([whether miniUPnPc API version is supported]) + AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ + @%:@include <miniupnpc/miniupnpc.h> + ]], [[ + #if MINIUPNPC_API_VERSION >= 10 + // Everything is okay + #else + # error miniUPnPc API version is too old + #endif + ]])],[ + AC_MSG_RESULT([yes]) + ],[ + AC_MSG_RESULT([no]) + AC_MSG_WARN([miniUPnPc API version < 10 is unsupported, disabling UPnP support.]) + have_miniupnpc=no + ]) + fi + CPPFLAGS="$TEMP_CPPFLAGS" fi dnl Check for libnatpmp (optional). if test "$use_natpmp" != "no"; then + TEMP_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $NATPMP_CPPFLAGS" AC_CHECK_HEADERS([natpmp.h], - [AC_CHECK_LIB([natpmp], [initnatpmp], [NATPMP_LIBS=-lnatpmp], [have_natpmp=no])], + [AC_CHECK_LIB([natpmp], [initnatpmp], [NATPMP_LIBS="$NATPMP_LIBS -lnatpmp"], [have_natpmp=no], [$NATPMP_LIBS])], [have_natpmp=no]) + CPPFLAGS="$TEMP_CPPFLAGS" fi if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench" = "nonononononono"; then @@ -1422,6 +1452,9 @@ if test "$use_boost" = "yes"; then AC_MSG_ERROR([only libbitcoinconsensus can be built without Boost]) fi + dnl we don't use multi_index serialization + BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_MULTI_INDEX_DISABLE_SERIALIZATION" + if test "$suppress_external_warnings" != "no"; then BOOST_CPPFLAGS=SUPPRESS_WARNINGS($BOOST_CPPFLAGS) fi @@ -1440,6 +1473,12 @@ if test "$use_external_signer" != "no"; then ;; *) AC_MSG_CHECKING([whether Boost.Process can be used]) + TEMP_CXXFLAGS="$CXXFLAGS" + dnl Boost 1.78 requires the following workaround. + dnl See: https://github.com/boostorg/process/issues/235 + CXXFLAGS="$CXXFLAGS -Wno-error=narrowing" + TEMP_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" TEMP_LDFLAGS="$LDFLAGS" dnl Boost 1.73 and older require the following workaround. LDFLAGS="$LDFLAGS $PTHREAD_CFLAGS" @@ -1447,6 +1486,8 @@ if test "$use_external_signer" != "no"; then [have_boost_process="yes"], [have_boost_process="no"]) LDFLAGS="$TEMP_LDFLAGS" + CPPFLAGS="$TEMP_CPPFLAGS" + CXXFLAGS="$TEMP_CXXFLAGS" AC_MSG_RESULT([$have_boost_process]) if test "$have_boost_process" = "yes"; then use_external_signer="yes" @@ -1494,7 +1535,7 @@ AM_CONDITIONAL([ENABLE_SYSCALL_SANDBOX], [test "$use_syscall_sandbox" != "no"]) dnl Check for reduced exports if test "$use_reduce_exports" = "yes"; then - AX_CHECK_COMPILE_FLAG([-fvisibility=hidden], [CXXFLAGS="$CXXFLAGS -fvisibility=hidden"], + AX_CHECK_COMPILE_FLAG([-fvisibility=hidden], [CORE_CXXFLAGS="$CORE_CXXFLAGS -fvisibility=hidden"], [AC_MSG_ERROR([Cannot set hidden symbol visibility. Use --disable-reduce-exports.])], [$CXXFLAG_WERROR]) AX_CHECK_LINK_FLAG([-Wl,--exclude-libs,ALL], [RELDFLAGS="-Wl,--exclude-libs,ALL"], [], [$LDFLAG_WERROR]) fi @@ -1509,9 +1550,9 @@ fi dnl libevent check if test "$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench" != "nonononono"; then - PKG_CHECK_MODULES([EVENT], [libevent >= 2.0.21], [use_libevent=yes], [AC_MSG_ERROR([libevent version 2.0.21 or greater not found.])]) + PKG_CHECK_MODULES([EVENT], [libevent >= 2.1.8], [use_libevent=yes], [AC_MSG_ERROR([libevent version 2.1.8 or greater not found.])]) if test "$TARGET_OS" != "windows"; then - PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads >= 2.0.21],, [AC_MSG_ERROR([libevent_pthreads version 2.0.21 or greater not found.])]) + PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads >= 2.1.8], [], [AC_MSG_ERROR([libevent_pthreads version 2.1.8 or greater not found.])]) fi if test "$suppress_external_warnings" != "no"; then @@ -1626,12 +1667,24 @@ AC_MSG_CHECKING([whether to build bitcoin-util]) AM_CONDITIONAL([BUILD_BITCOIN_UTIL], [test $build_bitcoin_util = "yes"]) AC_MSG_RESULT($build_bitcoin_util) +AC_MSG_CHECKING([whether to build experimental bitcoin-chainstate]) +if test "$build_experimental_kernel_lib" = "no"; then +AC_MSG_ERROR([experimental bitcoin-chainstate cannot be built without the experimental bitcoinkernel library. Use --with-experimental-kernel-lib]); +else + AM_CONDITIONAL([BUILD_BITCOIN_CHAINSTATE], [test $build_bitcoin_chainstate = "yes"]) +fi +AC_MSG_RESULT($build_bitcoin_chainstate) + AC_MSG_CHECKING([whether to build libraries]) AM_CONDITIONAL([BUILD_BITCOIN_LIBS], [test $build_bitcoin_libs = "yes"]) + if test "$build_bitcoin_libs" = "yes"; then AC_DEFINE([HAVE_CONSENSUS_LIB], [1], [Define this symbol if the consensus lib has been built]) AC_CONFIG_FILES([libbitcoinconsensus.pc:libbitcoinconsensus.pc.in]) fi + +AM_CONDITIONAL([BUILD_BITCOIN_KERNEL_LIB], [test "$build_experimental_kernel_lib" != "no" && ( test "$build_experimental_kernel_lib" = "yes" || test "$build_bitcoin_chainstate" = "yes" )]) + AC_MSG_RESULT($build_bitcoin_libs) AC_LANG_POP @@ -1688,7 +1741,7 @@ else AC_MSG_RESULT([$use_upnp_default]) AC_DEFINE_UNQUOTED([USE_UPNP],[$upnp_setting],[UPnP support not compiled if undefined, otherwise value (0 or 1) determines default state]) if test "$TARGET_OS" = "windows"; then - MINIUPNPC_CPPFLAGS="-DSTATICLIB -DMINIUPNP_STATICLIB" + MINIUPNPC_CPPFLAGS="$MINIUPNPC_CPPFLAGS -DSTATICLIB -DMINIUPNP_STATICLIB" fi else AC_MSG_RESULT([no]) @@ -1716,7 +1769,7 @@ else AC_MSG_RESULT($use_natpmp_default) AC_DEFINE_UNQUOTED([USE_NATPMP], [$natpmp_setting], [NAT-PMP support not compiled if undefined, otherwise value (0 or 1) determines default state]) if test "$TARGET_OS" = "windows"; then - NATPMP_CPPFLAGS="-DSTATICLIB -DNATPMP_STATICLIB" + NATPMP_CPPFLAGS="$NATPMP_CPPFLAGS -DSTATICLIB -DNATPMP_STATICLIB" fi else AC_MSG_RESULT([no]) @@ -1787,10 +1840,6 @@ if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_ AC_MSG_ERROR([No targets! Please specify at least one of: --with-utils --with-libs --with-daemon --with-gui --enable-bench or --enable-tests]) fi -if test "$enable_fuzz_binary" = "yes"; then - CHECK_RUNTIME_LIB -fi - AM_CONDITIONAL([TARGET_DARWIN], [test "$TARGET_OS" = "darwin"]) AM_CONDITIONAL([BUILD_DARWIN], [test "$BUILD_OS" = "darwin"]) AM_CONDITIONAL([TARGET_LINUX], [test "$TARGET_OS" = "linux"]) @@ -1845,11 +1894,15 @@ AC_SUBST(BITCOIN_GUI_NAME) AC_SUBST(BITCOIN_CLI_NAME) AC_SUBST(BITCOIN_TX_NAME) AC_SUBST(BITCOIN_UTIL_NAME) +AC_SUBST(BITCOIN_CHAINSTATE_NAME) AC_SUBST(BITCOIN_WALLET_TOOL_NAME) AC_SUBST(BITCOIN_MP_NODE_NAME) AC_SUBST(BITCOIN_MP_GUI_NAME) AC_SUBST(RELDFLAGS) +AC_SUBST(CORE_LDFLAGS) +AC_SUBST(CORE_CPPFLAGS) +AC_SUBST(CORE_CXXFLAGS) AC_SUBST(DEBUG_CPPFLAGS) AC_SUBST(WARN_CXXFLAGS) AC_SUBST(NOWARN_CXXFLAGS) @@ -1900,8 +1953,11 @@ AC_CONFIG_LINKS([contrib/devtools/security-check.py:contrib/devtools/security-ch AC_CONFIG_LINKS([contrib/devtools/symbol-check.py:contrib/devtools/symbol-check.py]) AC_CONFIG_LINKS([contrib/devtools/test-security-check.py:contrib/devtools/test-security-check.py]) AC_CONFIG_LINKS([contrib/devtools/test-symbol-check.py:contrib/devtools/test-symbol-check.py]) +AC_CONFIG_LINKS([contrib/devtools/iwyu/bitcoin.core.imp:contrib/devtools/iwyu/bitcoin.core.imp]) AC_CONFIG_LINKS([contrib/filter-lcov.py:contrib/filter-lcov.py]) AC_CONFIG_LINKS([contrib/macdeploy/background.tiff:contrib/macdeploy/background.tiff]) +AC_CONFIG_LINKS([src/.bear-tidy-config:src/.bear-tidy-config]) +AC_CONFIG_LINKS([src/.clang-tidy:src/.clang-tidy]) AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py]) AC_CONFIG_LINKS([test/fuzz/test_runner.py:test/fuzz/test_runner.py]) AC_CONFIG_LINKS([test/util/test_runner.py:test/util/test_runner.py]) @@ -1922,15 +1978,7 @@ LIBS_TEMP="$LIBS" unset LIBS LIBS="$LIBS_TEMP" -PKGCONFIG_PATH_TEMP="$PKG_CONFIG_PATH" -unset PKG_CONFIG_PATH -PKG_CONFIG_PATH="$PKGCONFIG_PATH_TEMP" - -PKGCONFIG_LIBDIR_TEMP="$PKG_CONFIG_LIBDIR" -unset PKG_CONFIG_LIBDIR -PKG_CONFIG_LIBDIR="$PKGCONFIG_LIBDIR_TEMP" - -ac_configure_args="${ac_configure_args} --disable-shared --with-pic --enable-benchmark=no --enable-module-recovery --enable-module-schnorrsig --enable-experimental" +ac_configure_args="${ac_configure_args} --disable-shared --with-pic --enable-benchmark=no --enable-module-recovery --enable-module-schnorrsig" AC_CONFIG_SUBDIRS([src/secp256k1]) AC_OUTPUT @@ -1981,9 +2029,9 @@ echo " build os = $build_os" echo echo " CC = $CC" echo " CFLAGS = $PTHREAD_CFLAGS $CFLAGS" -echo " CPPFLAGS = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CPPFLAGS" +echo " CPPFLAGS = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CORE_CPPFLAGS $CPPFLAGS" echo " CXX = $CXX" -echo " CXXFLAGS = $LTO_CXXFLAGS $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CXXFLAGS" -echo " LDFLAGS = $LTO_LDFLAGS $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS" +echo " CXXFLAGS = $LTO_CXXFLAGS $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CORE_CXXFLAGS $CXXFLAGS" +echo " LDFLAGS = $LTO_LDFLAGS $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $CORE_LDFLAGS $LDFLAGS" echo " ARFLAGS = $ARFLAGS" echo diff --git a/contrib/builder-keys/README.md b/contrib/builder-keys/README.md index 56bd87d0af..a6179d6012 100644 --- a/contrib/builder-keys/README.md +++ b/contrib/builder-keys/README.md @@ -19,9 +19,15 @@ gpg --refresh-keys To fetch keys of builders and active developers, feed the list of fingerprints of the primary keys into gpg: +On \*NIX: ```sh while read fingerprint keyholder_name; do gpg --keyserver hkps://keys.openpgp.org --recv-keys ${fingerprint}; done < ./keys.txt ``` +On Windows (requires Gpg4win >= 4.0.0): +``` +FOR /F "tokens=1" %i IN (keys.txt) DO gpg --keyserver hkps://keys.openpgp.org --recv-keys %i +``` + Add your key to the list if you provided Guix attestations for two major or minor releases of Bitcoin Core. diff --git a/contrib/builder-keys/keys.txt b/contrib/builder-keys/keys.txt index e8032f66ee..c70069b440 100644 --- a/contrib/builder-keys/keys.txt +++ b/contrib/builder-keys/keys.txt @@ -13,6 +13,7 @@ F20F56EF6A067F70E8A5C99FFF95FAA971697405 centaur (centaur) C060A6635913D98A3587D7DB1C2491FFEB0EF770 Cory Fields (cfields) BF6273FAEF7CC0BA1F562E50989F6B3048A116B5 Dev Random (devrandom) 6D3170C1DC2C6FD0AEEBCA6743811D1A26623924 Douglas Roark (droark) +948444FCE03B05BA5AB0591EC37B1C1D44C786EE Duncan Dean (dunxen) 1C6621605EC50319C463D56C7F81D87985D61612 Emanuele Cisbani (cisba) 9A1689B60D1B3CCE9262307A2F40A9BF167FBA47 Erik Mossberg (erkmos) D35176BE9264832E4ACA8986BF0792FBE95DC863 fivepiece (fivepiece) @@ -50,5 +51,6 @@ ED9BDF7AD6A55E232E84524257FF9BDBCC301009 Sjors Provoost (sjors) 9EDAFF80E080659604F4A76B2EBB056FD847F8A7 Stephan Oeste (Emzy) 6DEEF79B050C4072509B743F8C275BC595448867 Tomas Kanocz (KanoczTomas) AEC1884398647C47413C1C3FB1179EB7347DC10D Warren Togami (wtogami) +74E2DEF5D77260B98BC19438099BAD163C70FBFA Will Clark (will8clark) 79D00BAC68B56D422F945A8F8E3A8F3247DBCBBF Willy Ko (willyko) 71A3B16735405025D447E8F274810B012346C9A6 Wladimir J. van der Laan (laanwj) diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 79b0134adc..54b1a85588 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -90,6 +90,21 @@ example: BUILDDIR=$PWD/build contrib/devtools/gen-manpages.py ``` +gen-bitcoin-conf.sh +=================== + +Generates a bitcoin.conf file in `share/examples/` by parsing the output from `bitcoind --help`. This script is run during the +release process to include a bitcoin.conf with the release binaries and can also be run by users to generate a file locally. +When generating a file as part of the release process, make sure to commit the changes after running the script. + +With in-tree builds this tool can be run from any directory within the +repository. To use this tool with out-of-tree builds set `BUILDDIR`. For +example: + +```bash +BUILDDIR=$PWD/build contrib/devtools/gen-bitcoin-conf.sh +``` + security-check.py and test-security-check.py ============================================ diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py index 38f3df77c9..e20eb4b0d2 100755 --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -320,15 +320,13 @@ def get_most_recent_git_change_year(filename): ################################################################################ def read_file_lines(filename): - f = open(filename, 'r', encoding="utf8") - file_lines = f.readlines() - f.close() + with open(filename, 'r', encoding="utf8") as f: + file_lines = f.readlines() return file_lines def write_file_lines(filename, file_lines): - f = open(filename, 'w', encoding="utf8") - f.write(''.join(file_lines)) - f.close() + with open(filename, 'w', encoding="utf8") as f: + f.write(''.join(file_lines)) ################################################################################ # update header years execution diff --git a/contrib/devtools/gen-bitcoin-conf.sh b/contrib/devtools/gen-bitcoin-conf.sh new file mode 100755 index 0000000000..2ebbd42022 --- /dev/null +++ b/contrib/devtools/gen-bitcoin-conf.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C +TOPDIR=${TOPDIR:-$(git rev-parse --show-toplevel)} +BUILDDIR=${BUILDDIR:-$TOPDIR} +BINDIR=${BINDIR:-$BUILDDIR/src} +BITCOIND=${BITCOIND:-$BINDIR/bitcoind} +SHARE_EXAMPLES_DIR=${SHARE_EXAMPLES_DIR:-$TOPDIR/share/examples} +EXAMPLE_CONF_FILE=${EXAMPLE_CONF_FILE:-$SHARE_EXAMPLES_DIR/bitcoin.conf} + +[ ! -x "$BITCOIND" ] && echo "$BITCOIND not found or not executable." && exit 1 + +DIRTY="" +VERSION_OUTPUT=$($BITCOIND --version) +if [[ $VERSION_OUTPUT == *"dirty"* ]]; then + DIRTY="${DIRTY}${BITCOIND}\n" +fi + +if [ -n "$DIRTY" ] +then + echo -e "WARNING: $BITCOIND was built from a dirty tree.\n" + echo -e "To safely generate a bitcoin.conf file, please commit your changes to $BITCOIND, rebuild, then run this script again.\n" +fi + +echo 'Generating example bitcoin.conf file in share/examples/' + +# create the directory, if it doesn't exist +mkdir -p "${SHARE_EXAMPLES_DIR}" + +# create the header text +cat > "${EXAMPLE_CONF_FILE}" << 'EOF' +## +## bitcoin.conf configuration file. +## Generated by contrib/devtools/gen-bitcoin-conf.sh. +## +## Lines beginning with # are comments. +## All possible configuration options are provided. To use, copy this file +## to your data directory (default or specified by -datadir), uncomment +## options you would like to change, and save the file. +## + + +### Options +EOF + +# parse the output from bitcoind --help +# adding newlines is a bit funky to ensure portability for BSD +# see here for more details: https://stackoverflow.com/a/24575385 +${BITCOIND} --help \ + | sed '1,/Print this help message and exit/d' \ + | sed -E 's/^[[:space:]]{2}\-/#/' \ + | sed -E 's/^[[:space:]]{7}/# /' \ + | sed -E '/[=[:space:]]/!s/#.*$/&=1/' \ + | awk '/^#[a-z]/{x=$0;next}{if (NF==0) print x"\n",x="";else print}' \ + | sed 's,\(^[[:upper:]].*\)\:$,\ +### \1,' \ + | sed 's/[[:space:]]*$//' >> "${EXAMPLE_CONF_FILE}" + +# create the footer text +cat >> "${EXAMPLE_CONF_FILE}" << 'EOF' + +# [Sections] +# Most options will apply to all networks. To confine an option to a specific +# network, add it under the relevant section below. +# +# Note: If not specified under a network section, the options addnode, connect, +# port, bind, rpcport, rpcbind, and wallet will only apply to mainnet. + +# Options for mainnet +[main] + +# Options for testnet +[test] + +# Options for signet +[signet] + +# Options for regtest +[regtest] +EOF diff --git a/contrib/devtools/iwyu/bitcoin.core.imp b/contrib/devtools/iwyu/bitcoin.core.imp new file mode 100644 index 0000000000..ce7786f58c --- /dev/null +++ b/contrib/devtools/iwyu/bitcoin.core.imp @@ -0,0 +1,6 @@ +# Fixups / upstreamed changes +[ + { include: [ "<bits/termios-c_lflag.h>", private, "<termios.h>", public ] }, + { include: [ "<bits/termios-struct.h>", private, "<termios.h>", public ] }, + { include: [ "<bits/termios-tcflow.h>", private, "<termios.h>", public ] }, +] diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index e6a29b73b9..05c0af029e 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -12,10 +12,6 @@ from typing import List import lief #type:ignore -# temporary constant, to be replaced with lief.ELF.ARCH.RISCV -# https://github.com/lief-project/LIEF/pull/562 -LIEF_ELF_ARCH_RISCV = lief.ELF.ARCH(243) - def check_ELF_RELRO(binary) -> bool: ''' Check for read-only relocations. @@ -101,7 +97,6 @@ def check_ELF_separate_code(binary): for segment in binary.segments: if segment.type == lief.ELF.SEGMENT_TYPES.LOAD: for section in segment.sections: - assert(section.name not in flags_per_section) flags_per_section[section.name] = segment.flags # Spot-check ELF LOAD program header flags per section # If these sections exist, check them against the expected R/W/E flags @@ -222,7 +217,7 @@ CHECKS = { lief.ARCHITECTURES.ARM: BASE_ELF, lief.ARCHITECTURES.ARM64: BASE_ELF, lief.ARCHITECTURES.PPC: BASE_ELF, - LIEF_ELF_ARCH_RISCV: BASE_ELF, + lief.ARCHITECTURES.RISCV: BASE_ELF, }, lief.EXE_FORMATS.PE: { lief.ARCHITECTURES.X86: BASE_PE, @@ -250,12 +245,9 @@ if __name__ == '__main__': continue if arch == lief.ARCHITECTURES.NONE: - if binary.header.machine_type == LIEF_ELF_ARCH_RISCV: - arch = LIEF_ELF_ARCH_RISCV - else: - print(f'{filename}: unknown architecture') - retval = 1 - continue + print(f'{filename}: unknown architecture') + retval = 1 + continue failed: List[str] = [] for (name, func) in CHECKS[etype][arch]: diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 461132ae63..a419e392ee 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -15,10 +15,6 @@ from typing import List, Dict import lief #type:ignore -# temporary constant, to be replaced with lief.ELF.ARCH.RISCV -# https://github.com/lief-project/LIEF/pull/562 -LIEF_ELF_ARCH_RISCV = lief.ELF.ARCH(243) - # Debian 9 (Stretch) EOL: 2022. https://wiki.debian.org/DebianReleases#Production_Releases # # - g++ version 6.3.0 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=g%2B%2B) @@ -44,7 +40,7 @@ MAX_VERSIONS = { lief.ELF.ARCH.ARM: (2,18), lief.ELF.ARCH.AARCH64:(2,18), lief.ELF.ARCH.PPC64: (2,18), - LIEF_ELF_ARCH_RISCV: (2,27), + lief.ELF.ARCH.RISCV: (2,27), }, 'LIBATOMIC': (1,0), 'V': (0,5,0), # xkb (bitcoin-qt only) @@ -78,7 +74,7 @@ ELF_INTERPRETER_NAMES: Dict[lief.ELF.ARCH, Dict[lief.ENDIANNESS, str]] = { lief.ENDIANNESS.BIG: "/lib64/ld64.so.1", lief.ENDIANNESS.LITTLE: "/lib64/ld64.so.2", }, - LIEF_ELF_ARCH_RISCV: { + lief.ELF.ARCH.RISCV: { lief.ENDIANNESS.LITTLE: "/lib/ld-linux-riscv64-lp64d.so.1", }, } @@ -200,7 +196,7 @@ def check_exported_symbols(binary) -> bool: if not symbol.exported: continue name = symbol.name - if binary.header.machine_type == LIEF_ELF_ARCH_RISCV or name in IGNORE_EXPORTS: + if binary.header.machine_type == lief.ELF.ARCH.RISCV or name in IGNORE_EXPORTS: continue print(f'{binary.name}: export of symbol {name} not allowed!') ok = False diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py index e1a2ebc491..b4c112b266 100755 --- a/contrib/devtools/test-symbol-check.py +++ b/contrib/devtools/test-symbol-check.py @@ -187,7 +187,7 @@ class TestSymbolChecks(unittest.TestCase): executable = 'test3.exe' with open(source, 'w', encoding="utf8") as f: f.write(''' - #include <windows.h> + #include <combaseapi.h> int main() { diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 90289f9d40..af5607c710 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -75,7 +75,7 @@ crucial differences: 1. Since only Windows and macOS build outputs require codesigning, the `HOSTS` environment variable will have a sane default value of `x86_64-w64-mingw32 - x86_64-apple-darwin` instead of all the platforms. + x86_64-apple-darwin arm64-apple-darwin` instead of all the platforms. 2. The `guix-codesign` command ***requires*** a `DETACHED_SIGS_REPO` flag. * _**DETACHED_SIGS_REPO**_ diff --git a/contrib/guix/guix-attest b/contrib/guix/guix-attest index 6e12cbead7..b0ef28dc3f 100755 --- a/contrib/guix/guix-attest +++ b/contrib/guix/guix-attest @@ -19,8 +19,16 @@ source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" ################ check_tools cat env basename mkdir diff sort + if [ -z "$NO_SIGN" ]; then - check_tools gpg + # make it possible to override the gpg binary + GPG=${GPG:-gpg} + + # $GPG can contain extra arguments passed to the binary + # so let's check only the existence of arg[0] + # shellcheck disable=SC2206 + GPG_ARRAY=($GPG) + check_tools "${GPG_ARRAY[0]}" fi ################ @@ -90,7 +98,7 @@ if [ -z "${signer_name}" ]; then signer_name="$gpg_key_name" fi -if [ -z "$NO_SIGN" ] && ! gpg --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then +if [ -z "$NO_SIGN" ] && ! ${GPG} --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then echo "ERR: GPG can't seem to find any key named '${gpg_key_name}'" exit 1 fi @@ -239,11 +247,11 @@ mkdir -p "$outsigdir" echo "Signing SHA256SUMS to produce SHA256SUMS.asc" for i in *.SHA256SUMS; do if [ ! -e "$i".asc ]; then - gpg --detach-sign \ - --digest-algo sha256 \ - --local-user "$gpg_key_name" \ - --armor \ - --output "$i".asc "$i" + ${GPG} --detach-sign \ + --digest-algo sha256 \ + --local-user "$gpg_key_name" \ + --armor \ + --output "$i".asc "$i" else echo "Signature already there" fi diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index b38f91f3c7..74b24b9612 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -121,7 +121,7 @@ else fi ################ -# When building for darwin, the macOS SDK should exists +# When building for darwin, the macOS SDK should exist ################ for host in $HOSTS; do @@ -130,8 +130,9 @@ for host in $HOSTS; do OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=@@g')" if [ -e "$OSX_SDK" ]; then echo "Found macOS SDK at '${OSX_SDK}', using..." + break else - echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, exiting..." + echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, or define SDK_PATH environment variable. Exiting..." exit 1 fi ;; @@ -234,21 +235,6 @@ host_to_commonname() { # Determine the reference time used for determinism (overridable by environment) SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" -# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility -# across time. -time-machine() { - # shellcheck disable=SC2086 - guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=ae03f401381e956c4c41b4cf495cbde964fa43d0 \ - --cores="$JOBS" \ - --keep-failed \ - --fallback \ - ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ - ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ - -- "$@" -} - - # Precious directories are those which should not be cleaned between successive # guix builds depends_precious_dir_names='SOURCES_PATH BASE_CACHE SDK_PATH' diff --git a/contrib/guix/guix-codesign b/contrib/guix/guix-codesign index 2dd30bfa64..3279d431aa 100755 --- a/contrib/guix/guix-codesign +++ b/contrib/guix/guix-codesign @@ -152,10 +152,10 @@ outdir_for_host() { unsigned_tarball_for_host() { case "$1" in *mingw*) - echo "$(outdir_for_host "$1")/${DISTNAME}-win-unsigned.tar.gz" + echo "$(outdir_for_host "$1")/${DISTNAME}-win64-unsigned.tar.gz" ;; *darwin*) - echo "$(outdir_for_host "$1")/${DISTNAME}-osx-unsigned.tar.gz" + echo "$(outdir_for_host "$1")/${DISTNAME}-${1}-unsigned.tar.gz" ;; *) exit 1 @@ -222,20 +222,6 @@ JOBS="${JOBS:-$(nproc)}" # Determine the reference time used for determinism (overridable by environment) SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" -# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility -# across time. -time-machine() { - # shellcheck disable=SC2086 - guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=aa34d4d28dfe25ba47d5800d05000fb7221788c0 \ - --cores="$JOBS" \ - --keep-failed \ - --fallback \ - ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ - ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ - -- "$@" -} - # Make sure an output directory exists for our builds OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" mkdir -p "$OUTDIR_BASE" diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index e06a469338..cdf0020d4d 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -79,19 +79,6 @@ prepend_to_search_env_var() { export "${1}=${2}${!1:+:}${!1}" } -case "$HOST" in - *darwin*) - # When targeting darwin, zlib is required by native_libdmg-hfsplus. - zlib_store_path=$(store_path "zlib") - zlib_static_store_path=$(store_path "zlib" static) - - prepend_to_search_env_var LIBRARY_PATH "${zlib_static_store_path}/lib:${zlib_store_path}/lib" - prepend_to_search_env_var C_INCLUDE_PATH "${zlib_store_path}/include" - prepend_to_search_env_var CPLUS_INCLUDE_PATH "${zlib_store_path}/include" - prepend_to_search_env_var OBJC_INCLUDE_PATH "${zlib_store_path}/include" - prepend_to_search_env_var OBJCPLUS_INCLUDE_PATH "${zlib_store_path}/include" -esac - # Set environment variables to point the CROSS toolchain to the right # includes/libs for $HOST case "$HOST" in @@ -167,7 +154,6 @@ case "$HOST" in *linux*) glibc_dynamic_linker=$( case "$HOST" in - i686-linux-gnu) echo /lib/ld-linux.so.2 ;; x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; @@ -204,19 +190,12 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \ ${SOURCES_PATH+SOURCES_PATH="$SOURCES_PATH"} \ ${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} \ ${SDK_PATH+SDK_PATH="$SDK_PATH"} \ - i686_linux_CC=i686-linux-gnu-gcc \ - i686_linux_CXX=i686-linux-gnu-g++ \ - i686_linux_AR=i686-linux-gnu-ar \ - i686_linux_RANLIB=i686-linux-gnu-ranlib \ - i686_linux_NM=i686-linux-gnu-nm \ - i686_linux_STRIP=i686-linux-gnu-strip \ x86_64_linux_CC=x86_64-linux-gnu-gcc \ x86_64_linux_CXX=x86_64-linux-gnu-g++ \ x86_64_linux_AR=x86_64-linux-gnu-ar \ x86_64_linux_RANLIB=x86_64-linux-gnu-ranlib \ x86_64_linux_NM=x86_64-linux-gnu-nm \ x86_64_linux_STRIP=x86_64-linux-gnu-strip \ - qt_config_opts_i686_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \ qt_config_opts_x86_64_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \ FORCE_USE_SYSTEM_CLANG=1 @@ -340,18 +319,17 @@ mkdir -p "$DISTSRC" mkdir -p "unsigned-app-${HOST}" cp --target-directory="unsigned-app-${HOST}" \ osx_volname \ - contrib/macdeploy/detached-sig-{apply,create}.sh \ - "${BASEPREFIX}/${HOST}"/native/bin/dmg + contrib/macdeploy/detached-sig-create.sh mv --target-directory="unsigned-app-${HOST}" dist ( cd "unsigned-app-${HOST}" find . -print0 \ | sort --zero-terminated \ | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz" && exit 1 ) + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" && exit 1 ) ) - make deploy ${V:+V=1} OSX_DMG="${OUTDIR}/${DISTNAME}-osx-unsigned.dmg" + make deploy ${V:+V=1} OSX_DMG="${OUTDIR}/${DISTNAME}-${HOST}-unsigned.dmg" ;; esac ( @@ -390,6 +368,10 @@ mkdir -p "$DISTSRC" ;; esac + # copy over the example bitcoin.conf file. if contrib/devtools/gen-bitcoin-conf.sh + # has not been run before buildling, this file will be a stub + cp "${DISTSRC}/share/examples/bitcoin.conf" "${DISTNAME}/" + # Finally, deterministically produce {non-,}debug binary tarballs ready # for release case "$HOST" in @@ -423,8 +405,8 @@ mkdir -p "$DISTSRC" find "${DISTNAME}" -print0 \ | sort --zero-terminated \ | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin/osx64}.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin/osx64}.tar.gz" && exit 1 ) + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) ;; esac ) # $DISTSRC/installed @@ -439,8 +421,8 @@ mkdir -p "$DISTSRC" find . -print0 \ | sort --zero-terminated \ | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" && exit 1 ) + | gzip -9n > "${OUTDIR}/${DISTNAME}-win64-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-win64-unsigned.tar.gz" && exit 1 ) ) ;; esac diff --git a/contrib/guix/libexec/codesign.sh b/contrib/guix/libexec/codesign.sh index a8867ca351..9a5d3a1ce5 100755 --- a/contrib/guix/libexec/codesign.sh +++ b/contrib/guix/libexec/codesign.sh @@ -84,14 +84,11 @@ mkdir -p "$DISTSRC" # Apply detached codesignatures to dist/ (in-place) signapple apply dist/Bitcoin-Qt.app codesignatures/osx/dist - # Make an uncompressed DMG from dist/ + # Make a DMG from dist/ xorrisofs -D -l -V "$(< osx_volname)" -no-pad -r -dir-mode 0755 \ - -o uncompressed.dmg \ + -o "${OUTDIR}/${DISTNAME}-${HOST}.dmg" \ dist \ -- -volume_date all_file_dates ="$SOURCE_DATE_EPOCH" - - # Compress uncompressed.dmg and output to OUTDIR - ./dmg dmg uncompressed.dmg "${OUTDIR}/${DISTNAME}-osx-signed.dmg" ;; *) exit 1 diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index 1f287e9bbb..f24c120863 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -46,6 +46,22 @@ exit 1 fi ################ +# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility +# across time. +time-machine() { + # shellcheck disable=SC2086 + guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ + --commit=34e9eae68c9583acce5abc4100add3d88932a5ae \ + --cores="$JOBS" \ + --keep-failed \ + --fallback \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ + -- "$@" +} + + +################ # Set common variables ################ diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index d296eb9543..b61c2b8899 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -130,6 +130,7 @@ chain for " target " development.")) (license (package-license xgcc))))) (define base-gcc gcc-10) +(define base-linux-kernel-headers linux-libre-headers-5.15) ;; Building glibc with stack smashing protector first landed in glibc 2.25, use ;; this function to disable for older glibcs @@ -148,7 +149,7 @@ chain for " target " development.")) (define* (make-bitcoin-cross-toolchain target #:key (base-gcc-for-libc gcc-7) - (base-kernel-headers linux-libre-headers-4.9) + (base-kernel-headers base-linux-kernel-headers) (base-libc (make-glibc-without-ssp glibc-2.24)) (base-gcc (make-gcc-rpath-link base-gcc))) "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values @@ -162,9 +163,10 @@ desirable for building Bitcoin Core release binaries." (define (make-gcc-with-pthreads gcc) (package-with-extra-configure-variable gcc "--enable-threads" "posix")) -;; Required to support std::filesystem for mingw-w64 target. -(define (make-gcc-without-newlib gcc) - (package-with-extra-configure-variable gcc "--with-newlib" "no")) +(define (make-mingw-w64-cross-gcc cross-gcc) + (package-with-extra-patches cross-gcc + (search-our-patches "vmov-alignment.patch" + "gcc-broken-longjmp.patch"))) (define (make-mingw-pthreads-cross-toolchain target) "Create a cross-compilation toolchain package for TARGET" @@ -172,7 +174,7 @@ desirable for building Bitcoin Core release binaries." (pthreads-xlibc mingw-w64-x86_64-winpthreads) (pthreads-xgcc (make-gcc-with-pthreads (cross-gcc target - #:xgcc (make-gcc-without-newlib (make-ssp-fixed-gcc base-gcc)) + #:xgcc (make-ssp-fixed-gcc (make-mingw-w64-cross-gcc base-gcc)) #:xbinutils xbinutils #:libc pthreads-xlibc)))) ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and @@ -201,7 +203,7 @@ chain for " target " development.")) (define-public lief (package (name "python-lief") - (version "0.11.5") + (version "0.12.0") (source (origin (method git-fetch) @@ -211,7 +213,7 @@ chain for " target " development.")) (file-name (git-file-name name version)) (sha256 (base32 - "0qahjfg1n0x76ps2mbyljvws1l3qhkqvmxqbahps4qgywl2hbdkj")))) + "026jchj56q25v6gc0754dj9cj5hz5zaza8ij93y5ga94w20kzm9q")))) (build-system python-build-system) (native-inputs `(("cmake" ,cmake))) @@ -352,7 +354,7 @@ thus should be able to compile on most platforms where these exist.") #t))))))) (define-public python-certvalidator - (let ((commit "e5bdb4bfcaa09fa0af355eb8867d00dfeecba08c")) + (let ((commit "a145bf25eb75a9f014b3e7678826132efbba6213")) (package (name "python-certvalidator") (version (git-version "0.1" "1" commit)) @@ -365,7 +367,7 @@ thus should be able to compile on most platforms where these exist.") (file-name (git-file-name name commit)) (sha256 (base32 - "18pvxkvpkfkzgvfylv0kx65pmxfcv1hpsg03cip93krfvrrl4c75")))) + "1qw2k7xis53179lpqdqyylbcmp76lj7sagp883wmxg5i7chhc96k")))) (build-system python-build-system) (propagated-inputs `(("python-asn1crypto" ,python-asn1crypto) @@ -401,11 +403,6 @@ thus should be able to compile on most platforms where these exist.") (string-append indent "@unittest.skip(\"Disabled by Guix\")\n" line))) - (substitute* "tests/test_validate.py" - (("^(.*)def test_revocation_mode_soft" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) #t)) (replace 'check (lambda _ @@ -490,7 +487,7 @@ and endian independent.") (license license:expat))) (define-public python-signapple - (let ((commit "b084cbbf44d5330448ffce0c7d118f75781b64bd")) + (let ((commit "8a945a2e7583be2665cf3a6a89d665b70ecd1ab6")) (package (name "python-signapple") (version (git-version "0.1" "1" commit)) @@ -503,7 +500,7 @@ and endian independent.") (file-name (git-file-name name commit)) (sha256 (base32 - "0k7inccl2mzac3wq4asbr0kl8s4cghm8982z54kfascqg45shv01")))) + "0fr1hangvfyiwflca6jg5g8zvg3jc9qr7vd2c12ff89pznf38dlg")))) (build-system python-build-system) (propagated-inputs `(("python-asn1crypto" ,python-asn1crypto) @@ -511,8 +508,7 @@ and endian independent.") ("python-certvalidator" ,python-certvalidator) ("python-elfesteem" ,python-elfesteem) ("python-requests" ,python-requests) - ("python-macholib" ,python-macholib) - ("libcrypto" ,openssl))) + ("python-macholib" ,python-macholib))) ;; There are no tests, but attempting to run python setup.py test leads to ;; problems, just disable the test (arguments '(#:tests? #f)) @@ -578,8 +574,6 @@ inspecting signatures in Mach-O binaries.") bzip2 gzip xz - zlib - (list zlib "static") ;; Build tools gnu-make libtool @@ -593,24 +587,30 @@ inspecting signatures in Mach-O binaries.") ;; Git git ;; Tests - lief - ;; Native gcc 7 toolchain - gcc-toolchain-7 - (list gcc-toolchain-7 "static")) + lief) (let ((target (getenv "HOST"))) (cond ((string-suffix? "-mingw32" target) ;; Windows - (list zip + (list ;; Native GCC 10 toolchain + gcc-toolchain-10 + (list gcc-toolchain-10 "static") + zip (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") (make-nsis-for-gcc-10 nsis-x86_64) osslsigncode)) ((string-contains target "-linux-") - (list (cond ((string-contains target "riscv64-") + (list ;; Native GCC 7 toolchain + gcc-toolchain-7 + (list gcc-toolchain-7 "static") + (cond ((string-contains target "riscv64-") (make-bitcoin-cross-toolchain target #:base-libc glibc-2.27/bitcoin-patched - #:base-kernel-headers linux-libre-headers-4.19)) + #:base-kernel-headers base-linux-kernel-headers)) (else (make-bitcoin-cross-toolchain target))))) ((string-contains target "darwin") - (list clang-toolchain-10 binutils cmake xorriso python-signapple)) + (list ;; Native GCC 10 toolchain + gcc-toolchain-10 + (list gcc-toolchain-10 "static") + clang-toolchain-10 binutils cmake xorriso python-signapple)) (else '()))))) diff --git a/contrib/guix/patches/gcc-broken-longjmp.patch b/contrib/guix/patches/gcc-broken-longjmp.patch new file mode 100644 index 0000000000..1cfc0918b0 --- /dev/null +++ b/contrib/guix/patches/gcc-broken-longjmp.patch @@ -0,0 +1,68 @@ +commit eb5698897c52702498938592d7f76e67d126451f +Author: Eric Botcazou <ebotcazou@adacore.com> +Date: Wed May 5 22:48:51 2021 +0200 + + Fix PR target/100402 + + This is a regression for 64-bit Windows present from mainline down to the 9 + branch and introduced by the fix for PR target/99234. Again SEH, but with + a twist related to the way MinGW implements setjmp/longjmp, which turns out + to be piggybacked on SEH with recent versions of MinGW, i.e. the longjmp + performs a bona-fide unwinding of the stack, because it calls RtlUnwindEx + with the second argument initially passed to setjmp, which is the result of + __builtin_frame_address (0) in the MinGW header file: + + define setjmp(BUF) _setjmp((BUF), __builtin_frame_address (0)) + + This means that we directly expose the frame pointer to the SEH machinery + here (unlike with regular exception handling where we use an intermediate + CFA) and thus that we cannot do whatever we want with it. The old code + would leave it unaligned, i.e. not multiple of 16, whereas the new code + aligns it, but this breaks for some reason; at least it appears that a + .seh_setframe directive with 0 as second argument always works, so the + fix aligns it this way. + + gcc/ + PR target/100402 + * config/i386/i386.c (ix86_compute_frame_layout): For a SEH target, + always return the establisher frame for __builtin_frame_address (0). + gcc/testsuite/ + * gcc.c-torture/execute/20210505-1.c: New test. + +diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c +index 2f838840e96..06ad1b2274e 100644 +--- a/gcc/config/i386/i386.c ++++ b/gcc/config/i386/i386.c +@@ -6356,12 +6356,29 @@ ix86_compute_frame_layout (void) + area, see the SEH code in config/i386/winnt.c for the rationale. */ + frame->hard_frame_pointer_offset = frame->sse_reg_save_offset; + +- /* If we can leave the frame pointer where it is, do so. Also, return ++ /* If we can leave the frame pointer where it is, do so; however return + the establisher frame for __builtin_frame_address (0) or else if the +- frame overflows the SEH maximum frame size. */ ++ frame overflows the SEH maximum frame size. ++ ++ Note that the value returned by __builtin_frame_address (0) is quite ++ constrained, because setjmp is piggybacked on the SEH machinery with ++ recent versions of MinGW: ++ ++ # elif defined(__SEH__) ++ # if defined(__aarch64__) || defined(_ARM64_) ++ # define setjmp(BUF) _setjmp((BUF), __builtin_sponentry()) ++ # elif (__MINGW_GCC_VERSION < 40702) ++ # define setjmp(BUF) _setjmp((BUF), mingw_getsp()) ++ # else ++ # define setjmp(BUF) _setjmp((BUF), __builtin_frame_address (0)) ++ # endif ++ ++ and the second argument passed to _setjmp, if not null, is forwarded ++ to the TargetFrame parameter of RtlUnwindEx by longjmp (after it has ++ built an ExceptionRecord on the fly describing the setjmp buffer). */ + const HOST_WIDE_INT diff + = frame->stack_pointer_offset - frame->hard_frame_pointer_offset; +- if (diff <= 255) ++ if (diff <= 255 && !crtl->accesses_prior_frames) + { + /* The resulting diff will be a multiple of 16 lower than 255, + i.e. at most 240 as required by the unwind data structure. */ diff --git a/contrib/guix/patches/vmov-alignment.patch b/contrib/guix/patches/vmov-alignment.patch new file mode 100644 index 0000000000..072f76eafd --- /dev/null +++ b/contrib/guix/patches/vmov-alignment.patch @@ -0,0 +1,267 @@ +Description: Use unaligned VMOV instructions +Author: Stephen Kitt <skitt@debian.org> +Bug-Debian: https://bugs.debian.org/939559 + +Based on a patch originally by Claude Heiland-Allen <claude@mathr.co.uk> + +--- a/gcc/config/i386/sse.md ++++ b/gcc/config/i386/sse.md +@@ -1058,17 +1058,11 @@ + { + if (FLOAT_MODE_P (GET_MODE_INNER (<MODE>mode))) + { +- if (misaligned_operand (operands[1], <MODE>mode)) +- return "vmovu<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; +- else +- return "vmova<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; ++ return "vmovu<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; + } + else + { +- if (misaligned_operand (operands[1], <MODE>mode)) +- return "vmovdqu<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; +- else +- return "vmovdqa<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; ++ return "vmovdqu<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; + } + } + [(set_attr "type" "ssemov") +@@ -1184,17 +1178,11 @@ + { + if (FLOAT_MODE_P (GET_MODE_INNER (<MODE>mode))) + { +- if (misaligned_operand (operands[0], <MODE>mode)) +- return "vmovu<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; +- else +- return "vmova<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; ++ return "vmovu<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; + } + else + { +- if (misaligned_operand (operands[0], <MODE>mode)) +- return "vmovdqu<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; +- else +- return "vmovdqa<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; ++ return "vmovdqu<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; + } + } + [(set_attr "type" "ssemov") +@@ -7806,7 +7794,7 @@ + "TARGET_SSE && !(MEM_P (operands[0]) && MEM_P (operands[1]))" + "@ + %vmovlps\t{%1, %0|%q0, %1} +- %vmovaps\t{%1, %0|%0, %1} ++ %vmovups\t{%1, %0|%0, %1} + %vmovlps\t{%1, %d0|%d0, %q1}" + [(set_attr "type" "ssemov") + (set_attr "prefix" "maybe_vex") +@@ -13997,29 +13985,15 @@ + switch (<MODE>mode) + { + case E_V8DFmode: +- if (misaligned_operand (operands[2], <ssequartermode>mode)) +- return "vmovupd\t{%2, %x0|%x0, %2}"; +- else +- return "vmovapd\t{%2, %x0|%x0, %2}"; ++ return "vmovupd\t{%2, %x0|%x0, %2}"; + case E_V16SFmode: +- if (misaligned_operand (operands[2], <ssequartermode>mode)) +- return "vmovups\t{%2, %x0|%x0, %2}"; +- else +- return "vmovaps\t{%2, %x0|%x0, %2}"; ++ return "vmovups\t{%2, %x0|%x0, %2}"; + case E_V8DImode: +- if (misaligned_operand (operands[2], <ssequartermode>mode)) +- return which_alternative == 2 ? "vmovdqu64\t{%2, %x0|%x0, %2}" ++ return which_alternative == 2 ? "vmovdqu64\t{%2, %x0|%x0, %2}" + : "vmovdqu\t{%2, %x0|%x0, %2}"; +- else +- return which_alternative == 2 ? "vmovdqa64\t{%2, %x0|%x0, %2}" +- : "vmovdqa\t{%2, %x0|%x0, %2}"; + case E_V16SImode: +- if (misaligned_operand (operands[2], <ssequartermode>mode)) +- return which_alternative == 2 ? "vmovdqu32\t{%2, %x0|%x0, %2}" ++ return which_alternative == 2 ? "vmovdqu32\t{%2, %x0|%x0, %2}" + : "vmovdqu\t{%2, %x0|%x0, %2}"; +- else +- return which_alternative == 2 ? "vmovdqa32\t{%2, %x0|%x0, %2}" +- : "vmovdqa\t{%2, %x0|%x0, %2}"; + default: + gcc_unreachable (); + } +@@ -21225,63 +21199,27 @@ + switch (get_attr_mode (insn)) + { + case MODE_V16SF: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- return "vmovups\t{%1, %t0|%t0, %1}"; +- else +- return "vmovaps\t{%1, %t0|%t0, %1}"; ++ return "vmovups\t{%1, %t0|%t0, %1}"; + case MODE_V8DF: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- return "vmovupd\t{%1, %t0|%t0, %1}"; +- else +- return "vmovapd\t{%1, %t0|%t0, %1}"; ++ return "vmovupd\t{%1, %t0|%t0, %1}"; + case MODE_V8SF: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- return "vmovups\t{%1, %x0|%x0, %1}"; +- else +- return "vmovaps\t{%1, %x0|%x0, %1}"; ++ return "vmovups\t{%1, %x0|%x0, %1}"; + case MODE_V4DF: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- return "vmovupd\t{%1, %x0|%x0, %1}"; +- else +- return "vmovapd\t{%1, %x0|%x0, %1}"; ++ return "vmovupd\t{%1, %x0|%x0, %1}"; + case MODE_XI: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- { +- if (which_alternative == 2) +- return "vmovdqu\t{%1, %t0|%t0, %1}"; +- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) +- return "vmovdqu64\t{%1, %t0|%t0, %1}"; +- else +- return "vmovdqu32\t{%1, %t0|%t0, %1}"; +- } ++ if (which_alternative == 2) ++ return "vmovdqu\t{%1, %t0|%t0, %1}"; ++ else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) ++ return "vmovdqu64\t{%1, %t0|%t0, %1}"; + else +- { +- if (which_alternative == 2) +- return "vmovdqa\t{%1, %t0|%t0, %1}"; +- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) +- return "vmovdqa64\t{%1, %t0|%t0, %1}"; +- else +- return "vmovdqa32\t{%1, %t0|%t0, %1}"; +- } ++ return "vmovdqu32\t{%1, %t0|%t0, %1}"; + case MODE_OI: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- { +- if (which_alternative == 2) +- return "vmovdqu\t{%1, %x0|%x0, %1}"; +- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) +- return "vmovdqu64\t{%1, %x0|%x0, %1}"; +- else +- return "vmovdqu32\t{%1, %x0|%x0, %1}"; +- } ++ if (which_alternative == 2) ++ return "vmovdqu\t{%1, %x0|%x0, %1}"; ++ else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) ++ return "vmovdqu64\t{%1, %x0|%x0, %1}"; + else +- { +- if (which_alternative == 2) +- return "vmovdqa\t{%1, %x0|%x0, %1}"; +- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) +- return "vmovdqa64\t{%1, %x0|%x0, %1}"; +- else +- return "vmovdqa32\t{%1, %x0|%x0, %1}"; +- } ++ return "vmovdqu32\t{%1, %x0|%x0, %1}"; + default: + gcc_unreachable (); + } +--- a/gcc/config/i386/i386.c ++++ b/gcc/config/i386/i386.c +@@ -4981,13 +4981,13 @@ + switch (type) + { + case opcode_int: +- opcode = misaligned_p ? "vmovdqu32" : "vmovdqa32"; ++ opcode = "vmovdqu32"; + break; + case opcode_float: +- opcode = misaligned_p ? "vmovups" : "vmovaps"; ++ opcode = "vmovups"; + break; + case opcode_double: +- opcode = misaligned_p ? "vmovupd" : "vmovapd"; ++ opcode = "vmovupd"; + break; + } + } +@@ -4996,16 +4996,16 @@ + switch (scalar_mode) + { + case E_SFmode: +- opcode = misaligned_p ? "%vmovups" : "%vmovaps"; ++ opcode = "%vmovups"; + break; + case E_DFmode: +- opcode = misaligned_p ? "%vmovupd" : "%vmovapd"; ++ opcode = "%vmovupd"; + break; + case E_TFmode: + if (evex_reg_p) +- opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64"; ++ opcode = "vmovdqu64"; + else +- opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa"; ++ opcode = "%vmovdqu"; + break; + default: + gcc_unreachable (); +@@ -5017,48 +5017,32 @@ + { + case E_QImode: + if (evex_reg_p) +- opcode = (misaligned_p +- ? (TARGET_AVX512BW +- ? "vmovdqu8" +- : "vmovdqu64") +- : "vmovdqa64"); ++ opcode = TARGET_AVX512BW ? "vmovdqu8" : "vmovdqu64"; + else +- opcode = (misaligned_p +- ? (TARGET_AVX512BW +- ? "vmovdqu8" +- : "%vmovdqu") +- : "%vmovdqa"); ++ opcode = TARGET_AVX512BW ? "vmovdqu8" : "%vmovdqu"; + break; + case E_HImode: + if (evex_reg_p) +- opcode = (misaligned_p +- ? (TARGET_AVX512BW +- ? "vmovdqu16" +- : "vmovdqu64") +- : "vmovdqa64"); ++ opcode = TARGET_AVX512BW ? "vmovdqu16" : "vmovdqu64"; + else +- opcode = (misaligned_p +- ? (TARGET_AVX512BW +- ? "vmovdqu16" +- : "%vmovdqu") +- : "%vmovdqa"); ++ opcode = TARGET_AVX512BW ? "vmovdqu16" : "%vmovdqu"; + break; + case E_SImode: + if (evex_reg_p) +- opcode = misaligned_p ? "vmovdqu32" : "vmovdqa32"; ++ opcode = "vmovdqu32"; + else +- opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa"; ++ opcode = "%vmovdqu"; + break; + case E_DImode: + case E_TImode: + case E_OImode: + if (evex_reg_p) +- opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64"; ++ opcode = "vmovdqu64"; + else +- opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa"; ++ opcode = "%vmovdqu"; + break; + case E_XImode: +- opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64"; ++ opcode = "vmovdqu64"; + break; + default: + gcc_unreachable (); diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index 441b5da764..b72c7b0d08 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -20,49 +20,9 @@ from collections import namedtuple settings = {} -def hex_switchEndian(s): - """ Switches the endianness of a hex string (in pairs of hex chars) """ - pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)] - return b''.join(pairList[::-1]).decode() - -def uint32(x): - return x & 0xffffffff - -def bytereverse(x): - return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) | - (((x) >> 8) & 0x0000ff00) | ((x) >> 24) )) - -def bufreverse(in_buf): - out_words = [] - for i in range(0, len(in_buf), 4): - word = struct.unpack('@I', in_buf[i:i+4])[0] - out_words.append(struct.pack('@I', bytereverse(word))) - return b''.join(out_words) - -def wordreverse(in_buf): - out_words = [] - for i in range(0, len(in_buf), 4): - out_words.append(in_buf[i:i+4]) - out_words.reverse() - return b''.join(out_words) - -def calc_hdr_hash(blk_hdr): - hash1 = hashlib.sha256() - hash1.update(blk_hdr) - hash1_o = hash1.digest() - - hash2 = hashlib.sha256() - hash2.update(hash1_o) - hash2_o = hash2.digest() - - return hash2_o - def calc_hash_str(blk_hdr): - hash = calc_hdr_hash(blk_hdr) - hash = bufreverse(hash) - hash = wordreverse(hash) - hash_str = hash.hex() - return hash_str + blk_hdr_hash = hashlib.sha256(hashlib.sha256(blk_hdr).digest()).digest() + return blk_hdr_hash[::-1].hex() def get_blk_dt(blk_hdr): members = struct.unpack("<I", blk_hdr[68:68+4]) @@ -74,12 +34,12 @@ def get_blk_dt(blk_hdr): # When getting the list of block hashes, undo any byte reversals. def get_block_hashes(settings): blkindex = [] - f = open(settings['hashlist'], "r", encoding="utf8") - for line in f: - line = line.rstrip() - if settings['rev_hash_bytes'] == 'true': - line = hex_switchEndian(line) - blkindex.append(line) + with open(settings['hashlist'], "r", encoding="utf8") as f: + for line in f: + line = line.rstrip() + if settings['rev_hash_bytes'] == 'true': + line = bytes.fromhex(line)[::-1].hex() + blkindex.append(line) print("Read " + str(len(blkindex)) + " hashes") @@ -289,19 +249,18 @@ if __name__ == '__main__': print("Usage: linearize-data.py CONFIG-FILE") sys.exit(1) - f = open(sys.argv[1], encoding="utf8") - for line in f: - # skip comment lines - m = re.search(r'^\s*#', line) - if m: - continue - - # parse key=value lines - m = re.search(r'^(\w+)\s*=\s*(\S.*)$', line) - if m is None: - continue - settings[m.group(1)] = m.group(2) - f.close() + with open(sys.argv[1], encoding="utf8") as f: + for line in f: + # skip comment lines + m = re.search(r'^\s*#', line) + if m: + continue + + # parse key=value lines + m = re.search(r'^(\w+)\s*=\s*(\S.*)$', line) + if m is None: + continue + settings[m.group(1)] = m.group(2) # Force hash byte format setting to be lowercase to make comparisons easier. # Also place upfront in case any settings need to know about it. diff --git a/contrib/linearize/linearize-hashes.py b/contrib/linearize/linearize-hashes.py index fed6e665b8..5959300e74 100755 --- a/contrib/linearize/linearize-hashes.py +++ b/contrib/linearize/linearize-hashes.py @@ -17,11 +17,6 @@ import os.path settings = {} -def hex_switchEndian(s): - """ Switches the endianness of a hex string (in pairs of hex chars) """ - pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)] - return b''.join(pairList[::-1]).decode() - class BitcoinRPC: def __init__(self, host, port, username, password): authpair = "%s:%s" % (username, password) @@ -85,7 +80,7 @@ def get_block_hashes(settings, max_blocks_per_call=10000): sys.exit(1) assert(resp_obj['id'] == x) # assume replies are in-sequence if settings['rev_hash_bytes'] == 'true': - resp_obj['result'] = hex_switchEndian(resp_obj['result']) + resp_obj['result'] = bytes.fromhex(resp_obj['result'])[::-1].hex() print(resp_obj['result']) height += num_blocks @@ -103,19 +98,18 @@ if __name__ == '__main__': print("Usage: linearize-hashes.py CONFIG-FILE") sys.exit(1) - f = open(sys.argv[1], encoding="utf8") - for line in f: - # skip comment lines - m = re.search(r'^\s*#', line) - if m: - continue - - # parse key=value lines - m = re.search(r'^(\w+)\s*=\s*(\S.*)$', line) - if m is None: - continue - settings[m.group(1)] = m.group(2) - f.close() + with open(sys.argv[1], encoding="utf8") as f: + for line in f: + # skip comment lines + m = re.search(r'^\s*#', line) + if m: + continue + + # parse key=value lines + m = re.search(r'^(\w+)\s*=\s*(\S.*)$', line) + if m is None: + continue + settings[m.group(1)] = m.group(2) if 'host' not in settings: settings['host'] = '127.0.0.1' diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index ce69079e29..64e36f22b9 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -15,13 +15,16 @@ When complete, it will have produced `Bitcoin-Core.dmg`. A free Apple Developer Account is required to proceed. Our current macOS SDK -(`Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`) can be -extracted from +(`Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`) +can be extracted from [Xcode_12.2.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.2/Xcode_12.2.xip). + Alternatively, after logging in to your account go to 'Downloads', then 'More' and search for [`Xcode_12.2`](https://developer.apple.com/download/all/?q=Xcode%2012.2). + An Apple ID and cookies enabled for the hostname are needed to download this. -The `sha256sum` of the archive should be `28d352f8c14a43d9b8a082ac6338dc173cb153f964c6e8fb6ba389e5be528bd0`. + +The `sha256sum` of the downloaded XIP archive should be `28d352f8c14a43d9b8a082ac6338dc173cb153f964c6e8fb6ba389e5be528bd0`. After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip` archive. This makes the SDK less-trivial to extract on non-macOS machines. One @@ -55,7 +58,10 @@ previous stage) as the first argument. ./contrib/macdeploy/gen-sdk '/path/to/Xcode.app' ``` +The `sha256sum` of the generated TAR.GZ archive should be `df75d30ecafc429e905134333aeae56ac65fac67cb4182622398fd717df77619`. + ## Deterministic macOS DMG Notes + Working macOS DMGs are created in Linux by combining a recent `clang`, the Apple `binutils` (`ld`, `ar`, etc) and DMG authoring tools. @@ -89,16 +95,7 @@ redistributed. [`xorrisofs`](https://www.gnu.org/software/xorriso/) is used to create the DMG. -`xorrisofs` cannot compress DMGs, so afterwards, the DMG tool from the -`libdmg-hfsplus` project is used to compress it. There are several bugs in this -tool and its maintainer has seemingly abandoned the project. - -The DMG tool has the ability to create DMGs from scratch as well, but this functionality is -broken. Only the compression feature is currently used. Ideally, the creation could be fixed -and `xorrisofs` would no longer be necessary. - -Background images and other features can be added to DMG files by inserting a -`.DS_Store` during creation. +A background image is added to DMG files by inserting a `.DS_Store` during creation. As of OS X 10.9 Mavericks, using an Apple-blessed key to sign binaries is a requirement in order to satisfy the new Gatekeeper requirements. Because this private key cannot be diff --git a/contrib/macdeploy/detached-sig-apply.sh b/contrib/macdeploy/detached-sig-apply.sh deleted file mode 100755 index c7296387eb..0000000000 --- a/contrib/macdeploy/detached-sig-apply.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -# Copyright (c) 2014-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C -set -e - -UNSIGNED="$1" -SIGNATURE="$2" -ROOTDIR=dist -OUTDIR=signed-app -SIGNAPPLE=signapple - -if [ -z "$UNSIGNED" ]; then - echo "usage: $0 <unsigned app> <signature>" - exit 1 -fi - -if [ -z "$SIGNATURE" ]; then - echo "usage: $0 <unsigned app> <signature>" - exit 1 -fi - -${SIGNAPPLE} apply "${UNSIGNED}" "${SIGNATURE}" -mv ${ROOTDIR} ${OUTDIR} -echo "Signed: ${OUTDIR}" diff --git a/contrib/macdeploy/detached-sig-create.sh b/contrib/macdeploy/detached-sig-create.sh index 0d61ceb9df..f393331084 100755 --- a/contrib/macdeploy/detached-sig-create.sh +++ b/contrib/macdeploy/detached-sig-create.sh @@ -8,9 +8,11 @@ set -e ROOTDIR=dist BUNDLE="${ROOTDIR}/Bitcoin-Qt.app" +BINARY="${BUNDLE}/Contents/MacOS/Bitcoin-Qt" SIGNAPPLE=signapple TEMPDIR=sign.temp -OUT=signature-osx.tar.gz +ARCH=$(${SIGNAPPLE} info ${BINARY} | head -n 1 | cut -d " " -f 1) +OUT="signature-osx-${ARCH}.tar.gz" OUTROOT=osx/dist if [ -z "$1" ]; then diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk index ebef1d2db0..6efaaccb8e 100755 --- a/contrib/macdeploy/gen-sdk +++ b/contrib/macdeploy/gen-sdk @@ -8,6 +8,21 @@ import gzip import os import contextlib +# monkey-patch Python 3.8 and older to fix wrong TAR header handling +# see https://github.com/bitcoin/bitcoin/pull/24534 +# and https://github.com/python/cpython/pull/18080 for more info +if sys.version_info < (3, 9): + _old_create_header = tarfile.TarInfo._create_header + def _create_header(info, format, encoding, errors): + buf = _old_create_header(info, format, encoding, errors) + # replace devmajor/devminor with binary zeroes + buf = buf[:329] + bytes(16) + buf[345:] + # recompute checksum + chksum = tarfile.calc_chksums(buf)[0] + buf = buf[:-364] + bytes("%06o\0" % chksum, "ascii") + buf[-357:] + return buf + tarfile.TarInfo._create_header = staticmethod(_create_header) + @contextlib.contextmanager def cd(path): """Context manager that restores PWD even if an exception was raised.""" @@ -75,14 +90,21 @@ def run(): tarinfo.name = str(pathlib.Path(alt_base_dir, tarinfo.name)) if tarinfo.linkname and tarinfo.linkname.startswith("./"): tarinfo.linkname = str(pathlib.Path(alt_base_dir, tarinfo.linkname)) + # make metadata deterministic + tarinfo.mtime = 0 + tarinfo.uid, tarinfo.uname = 0, '' + tarinfo.gid, tarinfo.gname = 0, '' + # don't use isdir() as there are also executable files present + tarinfo.mode = 0o0755 if tarinfo.mode & 0o0100 else 0o0644 return tarinfo with cd(dir_to_add): + # recursion already adds entries in sorted order tarfp.add(".", recursive=True, filter=change_tarinfo_base) print("Creating output .tar.gz file...") with out_sdktgz_path.open("wb") as fp: with gzip.GzipFile(fileobj=fp, mode='wb', compresslevel=9, mtime=0) as gzf: - with tarfile.open(mode="w", fileobj=gzf) as tarfp: + with tarfile.open(mode="w", fileobj=gzf, format=tarfile.GNU_FORMAT) as tarfp: print("Adding MacOSX SDK {} files...".format(sdk_version)) tarfp_add_with_base_change(tarfp, sdk_dir, out_name) print("Adding libc++ headers...") diff --git a/contrib/macdeploy/macdeployqtplus b/contrib/macdeploy/macdeployqtplus index 0455a137f1..2420539b7c 100755 --- a/contrib/macdeploy/macdeployqtplus +++ b/contrib/macdeploy/macdeployqtplus @@ -211,7 +211,7 @@ def getFrameworks(binaryPath: str, verbose: int) -> List[FrameworkInfo]: return libraries def runInstallNameTool(action: str, *args): - installnametoolbin=os.getenv("INSTALLNAMETOOL", "install_name_tool") + installnametoolbin=os.getenv("INSTALL_NAME_TOOL", "install_name_tool") run([installnametoolbin, "-"+action] + list(args), check=True) def changeInstallName(oldName: str, newName: str, binaryPath: str, verbose: int): @@ -544,6 +544,22 @@ ds.close() if platform.system() == "Darwin": subprocess.check_call(f"codesign --deep --force --sign - {target}", shell=True) +print("+ Installing background.tiff +") + +bg_path = os.path.join('dist', '.background', 'background.tiff') +os.mkdir(os.path.dirname(bg_path)) + +tiff_path = os.path.join('contrib', 'macdeploy', 'background.tiff') +shutil.copy2(tiff_path, bg_path) + +# ------------------------------------------------ + +print("+ Generating symlink for /Applications +") + +os.symlink("/Applications", os.path.join('dist', "Applications")) + +# ------------------------------------------------ + if config.dmg is not None: print("+ Preparing .dmg disk image +") @@ -567,19 +583,6 @@ if config.dmg is not None: print("Attaching temp image...") output = run(["hdiutil", "attach", tempname, "-readwrite"], check=True, universal_newlines=True, stdout=PIPE).stdout - m = re.search(r"/Volumes/(.+$)", output) - disk_root = m.group(0) - - print("+ Applying fancy settings +") - - bg_path = os.path.join(disk_root, ".background", os.path.basename('background.tiff')) - os.mkdir(os.path.dirname(bg_path)) - if verbose: - print('background.tiff', "->", bg_path) - shutil.copy2('contrib/macdeploy/background.tiff', bg_path) - - os.symlink("/Applications", os.path.join(disk_root, "Applications")) - print("+ Finalizing .dmg disk image +") run(["hdiutil", "detach", f"/Volumes/{appname}"], universal_newlines=True) diff --git a/contrib/seeds/makeseeds.py b/contrib/seeds/makeseeds.py index 2b377f6c01..78eb04a836 100755 --- a/contrib/seeds/makeseeds.py +++ b/contrib/seeds/makeseeds.py @@ -10,18 +10,16 @@ import re import sys import dns.resolver import collections +from typing import List, Dict, Union NSEEDS=512 -MAX_SEEDS_PER_ASN=2 - -MIN_BLOCKS = 337600 - -# These are hosts that have been observed to be behaving strangely (e.g. -# aggressively connecting to every node). -with open("suspicious_hosts.txt", mode="r", encoding="utf-8") as f: - SUSPICIOUS_HOSTS = {s.strip() for s in f if s.strip()} +MAX_SEEDS_PER_ASN = { + 'ipv4': 2, + 'ipv6': 10, +} +MIN_BLOCKS = 730000 PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$") PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$") @@ -40,9 +38,13 @@ PATTERN_AGENT = re.compile( r"23.99" r")") -def parseline(line): +def parseline(line: str) -> Union[dict, None]: + """ Parses a line from `seeds_main.txt` into a dictionary of details for that line. + or `None`, if the line could not be parsed. + """ sline = line.split() if len(sline) < 11: + # line too short to be valid, skip it. return None m = PATTERN_IPV4.match(sline[0]) sortkey = None @@ -107,25 +109,26 @@ def parseline(line): 'sortkey': sortkey, } -def dedup(ips): - '''deduplicate by address,port''' +def dedup(ips: List[Dict]) -> List[Dict]: + """ Remove duplicates from `ips` where multiple ips share address and port. """ d = {} for ip in ips: d[ip['ip'],ip['port']] = ip return list(d.values()) -def filtermultiport(ips): - '''Filter out hosts with more nodes per IP''' +def filtermultiport(ips: List[Dict]) -> List[Dict]: + """ Filter out hosts with more nodes per IP""" hist = collections.defaultdict(list) for ip in ips: hist[ip['sortkey']].append(ip) return [value[0] for (key,value) in list(hist.items()) if len(value)==1] -def lookup_asn(net, ip): - ''' - Look up the asn for an IP (4 or 6) address by querying cymru.com, or None - if it could not be found. - ''' +def lookup_asn(net: str, ip: str) -> Union[int, None]: + """ Look up the asn for an `ip` address by querying cymru.com + on network `net` (e.g. ipv4 or ipv6). + + Returns in integer ASN or None if it could not be found. + """ try: if net == 'ipv4': ipaddr = ip @@ -147,20 +150,33 @@ def lookup_asn(net, ip): return None # Based on Greg Maxwell's seed_filter.py -def filterbyasn(ips, max_per_asn, max_per_net): +def filterbyasn(ips: List[Dict], max_per_asn: Dict, max_per_net: int) -> List[Dict]: + """ Prunes `ips` by + (a) trimming ips to have at most `max_per_net` ips from each net (e.g. ipv4, ipv6); and + (b) trimming ips to have at most `max_per_asn` ips from each asn in each net. + """ # Sift out ips by type ips_ipv46 = [ip for ip in ips if ip['net'] in ['ipv4', 'ipv6']] ips_onion = [ip for ip in ips if ip['net'] == 'onion'] # Filter IPv46 by ASN, and limit to max_per_net per network result = [] - net_count = collections.defaultdict(int) - asn_count = collections.defaultdict(int) - for ip in ips_ipv46: + net_count: Dict[str, int] = collections.defaultdict(int) + asn_count: Dict[int, int] = collections.defaultdict(int) + + for i, ip in enumerate(ips_ipv46): + if i % 10 == 0: + # give progress update + print(f"{i:6d}/{len(ips_ipv46)} [{100*i/len(ips_ipv46):04.1f}%]\r", file=sys.stderr, end='', flush=True) + if net_count[ip['net']] == max_per_net: + # do not add this ip as we already too many + # ips from this network continue asn = lookup_asn(ip['net'], ip['ip']) - if asn is None or asn_count[asn] == max_per_asn: + if asn is None or asn_count[asn] == max_per_asn[ip['net']]: + # do not add this ip as we already have too many + # ips from this ASN on this network continue asn_count[asn] += 1 net_count[ip['net']] += 1 @@ -170,35 +186,33 @@ def filterbyasn(ips, max_per_asn, max_per_net): result.extend(ips_onion[0:max_per_net]) return result -def ip_stats(ips): - hist = collections.defaultdict(int) +def ip_stats(ips: List[Dict]) -> str: + """ Format and return pretty string from `ips`. """ + hist: Dict[str, int] = collections.defaultdict(int) for ip in ips: if ip is not None: hist[ip['net']] += 1 - return '%6d %6d %6d' % (hist['ipv4'], hist['ipv6'], hist['onion']) + return f"{hist['ipv4']:6d} {hist['ipv6']:6d} {hist['onion']:6d}" def main(): lines = sys.stdin.readlines() ips = [parseline(line) for line in lines] print('\x1b[7m IPv4 IPv6 Onion Pass \x1b[0m', file=sys.stderr) - print('%s Initial' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Initial', file=sys.stderr) # Skip entries with invalid address. ips = [ip for ip in ips if ip is not None] - print('%s Skip entries with invalid address' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Skip entries with invalid address', file=sys.stderr) # Skip duplicates (in case multiple seeds files were concatenated) ips = dedup(ips) - print('%s After removing duplicates' % (ip_stats(ips)), file=sys.stderr) - # Skip entries from suspicious hosts. - ips = [ip for ip in ips if ip['ip'] not in SUSPICIOUS_HOSTS] - print('%s Skip entries from suspicious hosts' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} After removing duplicates', file=sys.stderr) # Enforce minimal number of blocks. ips = [ip for ip in ips if ip['blocks'] >= MIN_BLOCKS] - print('%s Enforce minimal number of blocks' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Enforce minimal number of blocks', file=sys.stderr) # Require service bit 1. ips = [ip for ip in ips if (ip['service'] & 1) == 1] - print('%s Require service bit 1' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Require service bit 1', file=sys.stderr) # Require at least 50% 30-day uptime for clearnet, 10% for onion. req_uptime = { 'ipv4': 50, @@ -206,18 +220,18 @@ def main(): 'onion': 10, } ips = [ip for ip in ips if ip['uptime'] > req_uptime[ip['net']]] - print('%s Require minimum uptime' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Require minimum uptime', file=sys.stderr) # Require a known and recent user agent. ips = [ip for ip in ips if PATTERN_AGENT.match(ip['agent'])] - print('%s Require a known and recent user agent' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Require a known and recent user agent', file=sys.stderr) # Sort by availability (and use last success as tie breaker) ips.sort(key=lambda x: (x['uptime'], x['lastsuccess'], x['ip']), reverse=True) # Filter out hosts with multiple bitcoin ports, these are likely abusive ips = filtermultiport(ips) - print('%s Filter out hosts with multiple bitcoin ports' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Filter out hosts with multiple bitcoin ports', file=sys.stderr) # Look up ASNs and limit results, both per ASN and globally. ips = filterbyasn(ips, MAX_SEEDS_PER_ASN, NSEEDS) - print('%s Look up ASNs and limit results per ASN and per net' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Look up ASNs and limit results per ASN and per net', file=sys.stderr) # Sort the results by IP address (for deterministic output). ips.sort(key=lambda x: (x['net'], x['sortkey'])) for ip in ips: diff --git a/contrib/seeds/suspicious_hosts.txt b/contrib/seeds/suspicious_hosts.txt deleted file mode 100644 index 13385cc816..0000000000 --- a/contrib/seeds/suspicious_hosts.txt +++ /dev/null @@ -1,16 +0,0 @@ -130.211.129.106 -148.251.238.178 -176.9.46.6 -178.63.107.226 -54.173.72.127 -54.174.10.182 -54.183.64.54 -54.194.231.211 -54.66.214.167 -54.66.220.137 -54.67.33.14 -54.77.251.214 -54.94.195.96 -54.94.200.247 -83.81.130.26 -88.198.17.7
\ No newline at end of file diff --git a/contrib/signet/miner b/contrib/signet/miner index 012bd6cc31..b366b98e2d 100755 --- a/contrib/signet/miner +++ b/contrib/signet/miner @@ -8,7 +8,7 @@ import base64 import json import logging import math -import os.path +import os import re import struct import sys @@ -493,10 +493,11 @@ def do_generate(args): logging.debug("Mining block delta=%s start=%s mine=%s", seconds_to_hms(mine_time-bestheader["time"]), mine_time, is_mine) mined_blocks += 1 psbt = generate_psbt(tmpl, reward_spk, blocktime=mine_time) - psbt_signed = json.loads(args.bcli("-stdin", "walletprocesspsbt", input=psbt.encode('utf8'))) + input_stream = os.linesep.join([psbt, "true", "ALL"]).encode('utf8') + psbt_signed = json.loads(args.bcli("-stdin", "walletprocesspsbt", input=input_stream)) if not psbt_signed.get("complete",False): logging.debug("Generated PSBT: %s" % (psbt,)) - sys.stderr.write("PSBT signing failed") + sys.stderr.write("PSBT signing failed\n") return 1 block, signet_solution = do_decode_psbt(psbt_signed["psbt"]) block = finish_block(block, signet_solution, args.grind_cmd) diff --git a/contrib/testgen/README.md b/contrib/testgen/README.md index fcc5a378e2..2f0288df16 100644 --- a/contrib/testgen/README.md +++ b/contrib/testgen/README.md @@ -2,7 +2,7 @@ Utilities to generate test vectors for the data-driven Bitcoin tests. -Usage: +To use inside a scripted-diff (or just execute directly): - PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py valid 70 > ../../src/test/data/key_io_valid.json - PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py invalid 70 > ../../src/test/data/key_io_invalid.json + ./gen_key_io_test_vectors.py valid 70 > ../../src/test/data/key_io_valid.json + ./gen_key_io_test_vectors.py invalid 70 > ../../src/test/data/key_io_invalid.json diff --git a/contrib/testgen/base58.py b/contrib/testgen/base58.py deleted file mode 100644 index 87341ccf96..0000000000 --- a/contrib/testgen/base58.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright (c) 2012-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' -Bitcoin base58 encoding and decoding. - -Based on https://bitcointalk.org/index.php?topic=1026.0 (public domain) -''' -import hashlib - -# for compatibility with following code... -class SHA256: - new = hashlib.sha256 - -if str != bytes: - # Python 3.x - def ord(c): - return c - def chr(n): - return bytes( (n,) ) - -__b58chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' -__b58base = len(__b58chars) -b58chars = __b58chars - -def b58encode(v): - """ encode v, which is a string of bytes, to base58. - """ - long_value = 0 - for (i, c) in enumerate(v[::-1]): - if isinstance(c, str): - c = ord(c) - long_value += (256**i) * c - - result = '' - while long_value >= __b58base: - div, mod = divmod(long_value, __b58base) - result = __b58chars[mod] + result - long_value = div - result = __b58chars[long_value] + result - - # Bitcoin does a little leading-zero-compression: - # leading 0-bytes in the input become leading-1s - nPad = 0 - for c in v: - if c == 0: - nPad += 1 - else: - break - - return (__b58chars[0]*nPad) + result - -def b58decode(v, length = None): - """ decode v into a string of len bytes - """ - long_value = 0 - for i, c in enumerate(v[::-1]): - pos = __b58chars.find(c) - assert pos != -1 - long_value += pos * (__b58base**i) - - result = bytes() - while long_value >= 256: - div, mod = divmod(long_value, 256) - result = chr(mod) + result - long_value = div - result = chr(long_value) + result - - nPad = 0 - for c in v: - if c == __b58chars[0]: - nPad += 1 - continue - break - - result = bytes(nPad) + result - if length is not None and len(result) != length: - return None - - return result - -def checksum(v): - """Return 32-bit checksum based on SHA256""" - return SHA256.new(SHA256.new(v).digest()).digest()[0:4] - -def b58encode_chk(v): - """b58encode a string, with 32-bit checksum""" - return b58encode(v + checksum(v)) - -def b58decode_chk(v): - """decode a base58 string, check and remove checksum""" - result = b58decode(v) - if result is None: - return None - if result[-4:] == checksum(result[:-4]): - return result[:-4] - else: - return None - -def get_bcaddress_version(strAddress): - """ Returns None if strAddress is invalid. Otherwise returns integer version of address. """ - addr = b58decode_chk(strAddress) - if addr is None or len(addr)!=21: - return None - version = addr[0] - return ord(version) - -if __name__ == '__main__': - # Test case (from http://gitorious.org/bitcoin/python-base58.git) - assert get_bcaddress_version('15VjRaDX9zpbA8LVnbrCAFzrVzN7ixHNsC') == 0 - _ohai = 'o hai'.encode('ascii') - _tmp = b58encode(_ohai) - assert _tmp == 'DYB3oMS' - assert b58decode(_tmp, 5) == _ohai - print("Tests passed") diff --git a/contrib/testgen/gen_key_io_test_vectors.py b/contrib/testgen/gen_key_io_test_vectors.py index eafaaaeceb..7bfb1d76a8 100755 --- a/contrib/testgen/gen_key_io_test_vectors.py +++ b/contrib/testgen/gen_key_io_test_vectors.py @@ -1,21 +1,21 @@ #!/usr/bin/env python3 -# Copyright (c) 2012-2021 The Bitcoin Core developers +# Copyright (c) 2012-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Generate valid and invalid base58/bech32(m) address and private key test vectors. - -Usage: - PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py valid 70 > ../../src/test/data/key_io_valid.json - PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py invalid 70 > ../../src/test/data/key_io_invalid.json ''' -# 2012 Wladimir J. van der Laan -# Released under MIT License -import os + from itertools import islice -from base58 import b58encode_chk, b58decode_chk, b58chars +import os import random -from segwit_addr import bech32_encode, decode_segwit_address, convertbits, CHARSET, Encoding +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), '../../test/functional')) + +from test_framework.address import base58_to_byte, byte_to_base58, b58chars # noqa: E402 +from test_framework.script import OP_0, OP_1, OP_2, OP_3, OP_16, OP_DUP, OP_EQUAL, OP_EQUALVERIFY, OP_HASH160, OP_CHECKSIG # noqa: E402 +from test_framework.segwit_addr import bech32_encode, decode_segwit_address, convertbits, CHARSET, Encoding # noqa: E402 # key types PUBKEY_ADDRESS = 0 @@ -29,16 +29,6 @@ PRIVKEY_TEST = 239 PRIVKEY_REGTEST = 239 # script -OP_0 = 0x00 -OP_1 = 0x51 -OP_2 = 0x52 -OP_3 = 0x53 -OP_16 = 0x60 -OP_DUP = 0x76 -OP_EQUAL = 0x87 -OP_EQUALVERIFY = 0x88 -OP_HASH160 = 0xa9 -OP_CHECKSIG = 0xac pubkey_prefix = (OP_DUP, OP_HASH160, 20) pubkey_suffix = (OP_EQUALVERIFY, OP_CHECKSIG) script_prefix = (OP_HASH160, 20) @@ -114,8 +104,10 @@ def is_valid(v): '''Check vector v for validity''' if len(set(v) - set(b58chars)) > 0: return is_valid_bech32(v) - result = b58decode_chk(v) - if result is None: + try: + payload, version = base58_to_byte(v) + result = bytes([version]) + payload + except ValueError: # thrown if checksum doesn't match return is_valid_bech32(v) for template in templates: prefix = bytearray(template[0]) @@ -135,18 +127,19 @@ def is_valid_bech32(v): def gen_valid_base58_vector(template): '''Generate valid base58 vector''' prefix = bytearray(template[0]) - payload = bytearray(os.urandom(template[1])) + payload = rand_bytes(size=template[1]) suffix = bytearray(template[2]) dst_prefix = bytearray(template[4]) dst_suffix = bytearray(template[5]) - rv = b58encode_chk(prefix + payload + suffix) + assert len(prefix) == 1 + rv = byte_to_base58(payload + suffix, prefix[0]) return rv, dst_prefix + payload + dst_suffix def gen_valid_bech32_vector(template): '''Generate valid bech32 vector''' hrp = template[0] witver = template[1] - witprog = bytearray(os.urandom(template[2])) + witprog = rand_bytes(size=template[2]) encoding = template[4] dst_prefix = bytearray(template[5]) rv = bech32_encode(encoding, hrp, [witver] + convertbits(witprog, 8, 5)) @@ -176,21 +169,22 @@ def gen_invalid_base58_vector(template): corrupt_suffix = randbool(0.2) if corrupt_prefix: - prefix = os.urandom(1) + prefix = rand_bytes(size=1) else: prefix = bytearray(template[0]) if randomize_payload_size: - payload = os.urandom(max(int(random.expovariate(0.5)), 50)) + payload = rand_bytes(size=max(int(random.expovariate(0.5)), 50)) else: - payload = os.urandom(template[1]) + payload = rand_bytes(size=template[1]) if corrupt_suffix: - suffix = os.urandom(len(template[2])) + suffix = rand_bytes(size=len(template[2])) else: suffix = bytearray(template[2]) - val = b58encode_chk(prefix + payload + suffix) + assert len(prefix) == 1 + val = byte_to_base58(payload + suffix, prefix[0]) if random.randint(0,10)<1: # line corruption if randbool(): # add random character to end val += random.choice(b58chars) @@ -206,7 +200,7 @@ def gen_invalid_bech32_vector(template): to_upper = randbool(0.1) hrp = template[0] witver = template[1] - witprog = bytearray(os.urandom(template[2])) + witprog = rand_bytes(size=template[2]) encoding = template[3] if no_data: @@ -236,6 +230,9 @@ def randbool(p = 0.5): '''Return True with P(p)''' return random.random() < p +def rand_bytes(*, size): + return bytearray(random.getrandbits(8) for _ in range(size)) + def gen_invalid_vectors(): '''Generate invalid test vectors''' # start with some manual edge-cases @@ -250,9 +247,9 @@ def gen_invalid_vectors(): yield val, if __name__ == '__main__': - import sys import json iters = {'valid':gen_valid_vectors, 'invalid':gen_invalid_vectors} + random.seed(42) try: uiter = iters[sys.argv[1]] except IndexError: diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index 99ca305fe7..6efe49254b 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -113,16 +113,6 @@ fun:GetCoin } { - Suppress boost warning - Memcheck:Leak - fun:_Znwm - ... - fun:_ZN5boost9unit_test9framework5state17execute_test_treeEmjPKNS2_23random_generator_helperE - fun:_ZN5boost9unit_test9framework3runEmb - fun:_ZN5boost9unit_test14unit_test_mainEPFbvEiPPc - fun:main -} -{ Suppress LogInstance still reachable memory warning Memcheck:Leak match-leak-kinds: reachable diff --git a/contrib/verify-commits/verify-commits.py b/contrib/verify-commits/verify-commits.py index 7e46c6fd47..2ff14c1f86 100755 --- a/contrib/verify-commits/verify-commits.py +++ b/contrib/verify-commits/verify-commits.py @@ -82,11 +82,16 @@ def main(): # get directory of this program and read data files dirname = os.path.dirname(os.path.abspath(__file__)) print("Using verify-commits data from " + dirname) - verified_root = open(dirname + "/trusted-git-root", "r", encoding="utf8").read().splitlines()[0] - verified_sha512_root = open(dirname + "/trusted-sha512-root-commit", "r", encoding="utf8").read().splitlines()[0] - revsig_allowed = open(dirname + "/allow-revsig-commits", "r", encoding="utf-8").read().splitlines() - unclean_merge_allowed = open(dirname + "/allow-unclean-merge-commits", "r", encoding="utf-8").read().splitlines() - incorrect_sha512_allowed = open(dirname + "/allow-incorrect-sha512-commits", "r", encoding="utf-8").read().splitlines() + with open(dirname + "/trusted-git-root", "r", encoding="utf8") as f: + verified_root = f.read().splitlines()[0] + with open(dirname + "/trusted-sha512-root-commit", "r", encoding="utf8") as f: + verified_sha512_root = f.read().splitlines()[0] + with open(dirname + "/allow-revsig-commits", "r", encoding="utf8") as f: + revsig_allowed = f.read().splitlines() + with open(dirname + "/allow-unclean-merge-commits", "r", encoding="utf8") as f: + unclean_merge_allowed = f.read().splitlines() + with open(dirname + "/allow-incorrect-sha512-commits", "r", encoding="utf8") as f: + incorrect_sha512_allowed = f.read().splitlines() # Set commit and branch and set variables current_commit = args.commit diff --git a/depends/Makefile b/depends/Makefile index 723509c81d..20f5f6b2c6 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -222,6 +222,9 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ -e 's|@RANLIB@|$(host_RANLIB)|' \ -e 's|@NM@|$(host_NM)|' \ -e 's|@STRIP@|$(host_STRIP)|' \ + -e 's|@OTOOL@|$(host_OTOOL)|' \ + -e 's|@INSTALL_NAME_TOOL@|$(host_INSTALL_NAME_TOOL)|' \ + -e 's|@DSYMUTIL@|$(host_DSYMUTIL)|' \ -e 's|@build_os@|$(build_os)|' \ -e 's|@host_os@|$(host_os)|' \ -e 's|@CFLAGS@|$(strip $(host_CFLAGS) $(host_$(release_type)_CFLAGS))|' \ @@ -271,7 +274,7 @@ clean-all: clean @rm -rf $(SOURCES_PATH) x86_64* i686* mips* arm* aarch64* powerpc* riscv32* riscv64* s390x* clean: - @rm -rf $(WORK_PATH) $(BASE_CACHE) $(BUILD) + @rm -rf $(WORK_PATH) $(BASE_CACHE) $(BUILD) *.log install: check-packages $(host_prefix)/share/config.site diff --git a/depends/README.md b/depends/README.md index f7647198c6..da2a74e0e7 100644 --- a/depends/README.md +++ b/depends/README.md @@ -113,7 +113,10 @@ The following can be set when running make: `make FOO=bar` - `BUILD_ID_SALT`: Optional salt to use when generating build package ids - `FORCE_USE_SYSTEM_CLANG`: (EXPERTS ONLY) When cross-compiling for macOS, use Clang found in the system's `$PATH` rather than the default prebuilt release of Clang - from llvm.org. Clang 8 or later is required. + from llvm.org. Clang 8 or later is required +- `LOG`: Use file-based logging for individual packages. During a package build its log file + resides in the `depends` directory, and the log file is printed out automatically in case + of build error. After successful build log files are moved along with package archives If some packages are not built, for example `make NO_WALLET=1`, the appropriate options will be passed to bitcoin's configure. In this case, `--disable-wallet`. diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk index ced01229dd..8ed82b276d 100644 --- a/depends/builders/darwin.mk +++ b/depends/builders/darwin.mk @@ -6,6 +6,7 @@ build_darwin_STRIP:=$(shell xcrun -f strip) build_darwin_OTOOL:=$(shell xcrun -f otool) build_darwin_NM:=$(shell xcrun -f nm) build_darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool) +build_darwin_DSYMUTIL:=$(shell xcrun -f dsymutil) build_darwin_SHA256SUM=shasum -a 256 build_darwin_DOWNLOAD=curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o @@ -19,6 +20,7 @@ darwin_LIBTOOL:=$(shell xcrun -f libtool) darwin_OTOOL:=$(shell xcrun -f otool) darwin_NM:=$(shell xcrun -f nm) darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool) +darwin_DSYMUTIL:=$(shell xcrun -f dsymutil) darwin_native_binutils= darwin_native_toolchain= diff --git a/depends/builders/default.mk b/depends/builders/default.mk index 0370fb9acb..cc6dec66c2 100644 --- a/depends/builders/default.mk +++ b/depends/builders/default.mk @@ -5,15 +5,13 @@ default_build_TAR = tar default_build_RANLIB = ranlib default_build_STRIP = strip default_build_NM = nm -default_build_OTOOL = otool -default_build_INSTALL_NAME_TOOL = install_name_tool define add_build_tool_func build_$(build_os)_$1 ?= $$(default_build_$1) build_$(build_arch)_$(build_os)_$1 ?= $$(build_$(build_os)_$1) build_$1=$$(build_$(build_arch)_$(build_os)_$1) endef -$(foreach var,CC CXX AR TAR RANLIB NM STRIP SHA256SUM DOWNLOAD OTOOL INSTALL_NAME_TOOL,$(eval $(call add_build_tool_func,$(var)))) +$(foreach var,CC CXX AR TAR RANLIB NM STRIP SHA256SUM DOWNLOAD OTOOL INSTALL_NAME_TOOL DSYMUTIL,$(eval $(call add_build_tool_func,$(var)))) define add_build_flags_func build_$(build_arch)_$(build_os)_$1 += $(build_$(build_os)_$1) build_$1=$$(build_$(build_arch)_$(build_os)_$1) diff --git a/depends/config.site.in b/depends/config.site.in index 95e6ae85cf..03dabeea0a 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -78,7 +78,6 @@ if test "@host_os@" = darwin; then BREW=no fi -PATH="${depends_prefix}/native/bin:${PATH}" PKG_CONFIG="$(which pkg-config) --static" # These two need to remain exported because pkg-config does not see them @@ -115,6 +114,28 @@ if test -n "@NM@"; then ac_cv_path_ac_pt_NM="${NM}" fi +if test -n "@STRIP@"; then + STRIP="@STRIP@" + ac_cv_path_ac_pt_STRIP="${STRIP}" +fi + +if test "@host_os@" = darwin; then + if test -n "@OTOOL@"; then + OTOOL="@OTOOL@" + ac_cv_path_ac_pt_OTOOL="${OTOOL}" + fi + + if test -n "@INSTALL_NAME_TOOL@"; then + INSTALL_NAME_TOOL="@INSTALL_NAME_TOOL@" + ac_cv_path_ac_pt_INSTALL_NAME_TOOL="${INSTALL_NAME_TOOL}" + fi + + if test -n "@DSYMUTIL@"; then + DSYMUTIL="@DSYMUTIL@" + ac_cv_path_ac_pt_DSYMUTIL="${DSYMUTIL}" + fi +fi + if test -n "@debug@"; then enable_reduce_exports=no fi diff --git a/depends/funcs.mk b/depends/funcs.mk index c4ad109c69..a00f380236 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -67,6 +67,7 @@ $(1)_cached_checksum:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_bui $(1)_patch_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)/.patches-$($(1)_build_id) $(1)_prefixbin:=$($($(1)_type)_prefix)/bin/ $(1)_cached:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz +$(1)_build_log:=$(BASEDIR)/$(1)-$($(1)_version)-$($(1)_build_id).log $(1)_all_sources=$($(1)_file_name) $($(1)_extra_sources) #stamps @@ -85,7 +86,7 @@ $(1)_download_path_fixed=$(subst :,\:,$$($(1)_download_path)) # The default behavior for tar will try to set ownership when running as uid 0 and may not succeed, --no-same-owner disables this behavior $(1)_fetch_cmds ?= $(call fetch_file,$(1),$(subst \:,:,$$($(1)_download_path_fixed)),$$($(1)_download_file),$($(1)_file_name),$($(1)_sha256_hash)) $(1)_extract_cmds ?= mkdir -p $$($(1)_extract_dir) && echo "$$($(1)_sha256_hash) $$($(1)_source)" > $$($(1)_extract_dir)/.$$($(1)_file_name).hash && $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$$($(1)_file_name).hash && $(build_TAR) --no-same-owner --strip-components=1 -xf $$($(1)_source) -$(1)_preprocess_cmds ?= +$(1)_preprocess_cmds ?= true $(1)_build_cmds ?= $(1)_config_cmds ?= $(1)_stage_cmds ?= @@ -174,7 +175,7 @@ $(1)_cmake=env CC="$$($(1)_cc)" \ CXX="$$($(1)_cxx)" \ CXXFLAGS="$$($(1)_cppflags) $$($(1)_cxxflags)" \ LDFLAGS="$$($(1)_ldflags)" \ - cmake -DCMAKE_INSTALL_PREFIX:PATH="$$($($(1)_type)_prefix)" + cmake -DCMAKE_INSTALL_PREFIX:PATH="$$($($(1)_type)_prefix)" $$($(1)_cmake_opts) ifeq ($($(1)_type),build) $(1)_cmake += -DCMAKE_INSTALL_RPATH:PATH="$$($($(1)_type)_prefix)/lib" else @@ -187,44 +188,48 @@ endif endef define int_add_cmds +ifneq ($(LOG),) +$(1)_logging = >>$$($(1)_build_log) 2>&1 || { if test -f $$($(1)_build_log); then cat $$($(1)_build_log); fi; exit 1; } +endif + $($(1)_fetched): mkdir -p $$(@D) $(SOURCES_PATH) rm -f $$@ touch $$@ - cd $$(@D); $(call $(1)_fetch_cmds,$(1)) + cd $$(@D); $($(1)_fetch_cmds) cd $($(1)_source_dir); $(foreach source,$($(1)_all_sources),$(build_SHA256SUM) $(source) >> $$(@);) touch $$@ $($(1)_extracted): | $($(1)_fetched) echo Extracting $(1)... mkdir -p $$(@D) - cd $$(@D); $(call $(1)_extract_cmds,$(1)) + cd $$(@D); $($(1)_extract_cmds) touch $$@ $($(1)_preprocessed): | $($(1)_extracted) echo Preprocessing $(1)... mkdir -p $$(@D) $($(1)_patch_dir) $(foreach patch,$($(1)_patches),cd $(PATCHES_PATH)/$(1); cp $(patch) $($(1)_patch_dir) ;) - cd $$(@D); $(call $(1)_preprocess_cmds, $(1)) + { cd $$(@D); $($(1)_preprocess_cmds); } $$($(1)_logging) touch $$@ $($(1)_configured): | $($(1)_dependencies) $($(1)_preprocessed) echo Configuring $(1)... rm -rf $(host_prefix); mkdir -p $(host_prefix)/lib; cd $(host_prefix); $(foreach package,$($(1)_all_dependencies), $(build_TAR) --no-same-owner -xf $($(package)_cached); ) mkdir -p $$(@D) - +cd $$(@D); $($(1)_config_env) $(call $(1)_config_cmds, $(1)) + +{ cd $$(@D); $($(1)_config_env) $($(1)_config_cmds); } $$($(1)_logging) touch $$@ $($(1)_built): | $($(1)_configured) echo Building $(1)... mkdir -p $$(@D) - +cd $$(@D); $($(1)_build_env) $(call $(1)_build_cmds, $(1)) + +{ cd $$(@D); $($(1)_build_env) $($(1)_build_cmds); } $$($(1)_logging) touch $$@ $($(1)_staged): | $($(1)_built) echo Staging $(1)... mkdir -p $($(1)_staging_dir)/$(host_prefix) - cd $($(1)_build_dir); $($(1)_stage_env) $(call $(1)_stage_cmds, $(1)) + +{ cd $($(1)_build_dir); $($(1)_stage_env) $($(1)_stage_cmds); } $$($(1)_logging) rm -rf $($(1)_extract_dir) touch $$@ $($(1)_postprocessed): | $($(1)_staged) echo Postprocessing $(1)... - cd $($(1)_staging_prefix_dir); $(call $(1)_postprocess_cmds) + cd $($(1)_staging_prefix_dir); $($(1)_postprocess_cmds) touch $$@ $($(1)_cached): | $($(1)_dependencies) $($(1)_postprocessed) echo Caching $(1)... @@ -233,6 +238,7 @@ $($(1)_cached): | $($(1)_dependencies) $($(1)_postprocessed) rm -rf $$(@D) && mkdir -p $$(@D) mv $$($(1)_staging_dir)/$$(@F) $$(@) rm -rf $($(1)_staging_dir) + if test -f $($(1)_build_log); then mv $($(1)_build_log) $$(@D); fi $($(1)_cached_checksum): $($(1)_cached) cd $$(@D); $(build_SHA256SUM) $$(<F) > $$(@) diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 6bf30b499a..bf9b7625f2 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -38,7 +38,7 @@ clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++") clang_resource_dir=$(shell clang -print-resource-dir) endif -cctools_TOOLS=AR RANLIB STRIP NM LIBTOOL OTOOL INSTALL_NAME_TOOL +cctools_TOOLS=AR RANLIB STRIP NM LIBTOOL OTOOL INSTALL_NAME_TOOL DSYMUTIL # Make-only lowercase function lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1)))))))))))))))))))))))))) diff --git a/depends/hosts/default.mk b/depends/hosts/default.mk index 258619a9d0..ea60f025de 100644 --- a/depends/hosts/default.mk +++ b/depends/hosts/default.mk @@ -8,8 +8,6 @@ default_host_AR = $(host_toolchain)ar default_host_RANLIB = $(host_toolchain)ranlib default_host_STRIP = $(host_toolchain)strip default_host_LIBTOOL = $(host_toolchain)libtool -default_host_INSTALL_NAME_TOOL = $(host_toolchain)install_name_tool -default_host_OTOOL = $(host_toolchain)otool default_host_NM = $(host_toolchain)nm define add_host_tool_func @@ -35,5 +33,5 @@ host_$1 = $$($(host_arch)_$(host_os)_$1) host_$(release_type)_$1 = $$($(host_arch)_$(host_os)_$(release_type)_$1) endef -$(foreach tool,CC CXX AR RANLIB STRIP NM LIBTOOL OTOOL INSTALL_NAME_TOOL,$(eval $(call add_host_tool_func,$(tool)))) +$(foreach tool,CC CXX AR RANLIB STRIP NM LIBTOOL OTOOL INSTALL_NAME_TOOL DSYMUTIL,$(eval $(call add_host_tool_func,$(tool)))) $(foreach flags,CFLAGS CXXFLAGS CPPFLAGS LDFLAGS, $(eval $(call add_host_flags_func,$(flags)))) diff --git a/depends/hosts/mingw32.mk b/depends/hosts/mingw32.mk index 2f370d2b87..48020d71af 100644 --- a/depends/hosts/mingw32.mk +++ b/depends/hosts/mingw32.mk @@ -13,4 +13,4 @@ mingw32_debug_CXXFLAGS=$(mingw32_debug_CFLAGS) mingw32_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC -mingw_cmake_system=Windows +mingw32_cmake_system=Windows diff --git a/depends/hosts/openbsd.mk b/depends/hosts/openbsd.mk index dc8393e04c..5988f24bff 100644 --- a/depends/hosts/openbsd.mk +++ b/depends/hosts/openbsd.mk @@ -1,11 +1,11 @@ openbsd_CFLAGS=-pipe -openbsd_CFLAGS_CXXFLAGS=$(openbsd_CFLAGS) +openbsd_CXXFLAGS=$(openbsd_CFLAGS) -openbsd_CFLAGS_release_CFLAGS=-O2 -openbsd_CFLAGS_release_CXXFLAGS=$(openbsd_release_CFLAGS) +openbsd_release_CFLAGS=-O2 +openbsd_release_CXXFLAGS=$(openbsd_release_CFLAGS) -openbsd_CFLAGS_debug_CFLAGS=-O1 -openbsd_CFLAGS_debug_CXXFLAGS=$(openbsd_debug_CFLAGS) +openbsd_debug_CFLAGS=-O1 +openbsd_debug_CXXFLAGS=$(openbsd_debug_CFLAGS) ifeq (86,$(findstring 86,$(build_arch))) i686_openbsd_CC=clang -m32 diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index 748ed510c1..1efe6220d3 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -36,5 +36,6 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/*.la + rm lib/*.la && \ + rm include/ev*.h endef diff --git a/depends/packages/libmultiprocess.mk b/depends/packages/libmultiprocess.mk index 40ab3c68ea..9b66207fc5 100644 --- a/depends/packages/libmultiprocess.mk +++ b/depends/packages/libmultiprocess.mk @@ -4,9 +4,19 @@ $(package)_download_path=$(native_$(package)_download_path) $(package)_file_name=$(native_$(package)_file_name) $(package)_sha256_hash=$(native_$(package)_sha256_hash) $(package)_dependencies=native_$(package) capnp +ifneq ($(host),$(build)) +$(package)_dependencies += native_capnp +endif + +define $(package)_set_vars := +ifneq ($(host),$(build)) +$(package)_cmake_opts := -DCAPNP_EXECUTABLE="$$(native_capnp_prefixbin)/capnp" +$(package)_cmake_opts += -DCAPNPC_CXX_EXECUTABLE="$$(native_capnp_prefixbin)/capnpc-c++" +endif +endef define $(package)_config_cmds - $($(package)_cmake) + $($(package)_cmake) . endef define $(package)_build_cmds diff --git a/depends/packages/native_libdmg-hfsplus.mk b/depends/packages/native_libdmg-hfsplus.mk deleted file mode 100644 index c7c8adef41..0000000000 --- a/depends/packages/native_libdmg-hfsplus.mk +++ /dev/null @@ -1,24 +0,0 @@ -package=native_libdmg-hfsplus -$(package)_version=7ac55ec64c96f7800d9818ce64c79670e7f02b67 -$(package)_download_path=https://github.com/planetbeing/libdmg-hfsplus/archive -$(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=56fbdc48ec110966342f0ecddd6f8f89202f4143ed2a3336e42bbf88f940850c -$(package)_build_subdir=build -$(package)_patches=remove-libcrypto-dependency.patch - -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/remove-libcrypto-dependency.patch && \ - mkdir build -endef - -define $(package)_config_cmds - $($(package)_cmake) -DCMAKE_C_FLAGS="$$($(1)_cflags) -Wl,--build-id=none" -DCMAKE_SKIP_RPATH="ON" -DCMAKE_EXE_LINKER_FLAGS="-static" -DCMAKE_FIND_LIBRARY_SUFFIXES=".a" .. -endef - -define $(package)_build_cmds - $(MAKE) -C dmg -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) -C dmg install -endef diff --git a/depends/packages/native_libmultiprocess.mk b/depends/packages/native_libmultiprocess.mk index 14653ce9fb..6e600c5720 100644 --- a/depends/packages/native_libmultiprocess.mk +++ b/depends/packages/native_libmultiprocess.mk @@ -6,7 +6,7 @@ $(package)_sha256_hash=9f8b055c8bba755dc32fe799b67c20b91e7b13e67cadafbc54c0f1def $(package)_dependencies=native_capnp define $(package)_config_cmds - $($(package)_cmake) + $($(package)_cmake) . endef define $(package)_build_cmds diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 991db7f46e..998cc0221c 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -26,7 +26,7 @@ usdt_linux_packages=systemtap darwin_native_packages = native_ds_store native_mac_alias ifneq ($(build_os),darwin) -darwin_native_packages += native_cctools native_libtapi native_libdmg-hfsplus +darwin_native_packages += native_cctools native_libtapi ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) darwin_native_packages+= native_clang diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 9cdfd21d2c..2bc3a81430 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -1,25 +1,32 @@ package=qt -$(package)_version=5.15.2 +$(package)_version=5.15.3 $(package)_download_path=https://download.qt.io/official_releases/qt/5.15/$($(package)_version)/submodules -$(package)_suffix=everywhere-src-$($(package)_version).tar.xz +$(package)_suffix=everywhere-opensource-src-$($(package)_version).tar.xz $(package)_file_name=qtbase-$($(package)_suffix) -$(package)_sha256_hash=909fad2591ee367993a75d7e2ea50ad4db332f05e1c38dd7a5a274e156a4e0f8 +$(package)_sha256_hash=26394ec9375d52c1592bd7b689b1619c6b8dbe9b6f91fdd5c355589787f3a0b6 $(package)_linux_dependencies=freetype fontconfig libxcb libxkbcommon libxcb_util libxcb_util_render libxcb_util_keysyms libxcb_util_image libxcb_util_wm $(package)_qt_libs=corelib network widgets gui plugins testlib $(package)_linguist_tools = lrelease lupdate lconvert -$(package)_patches = qt.pro qttools_src.pro -$(package)_patches += fix_qt_pkgconfig.patch mac-qmake.conf fix_no_printer.patch no-xlib.patch -$(package)_patches += dont_use_avx_android_x86_64.patch dont_hardcode_x86_64.patch fix_montery_include.patch -$(package)_patches += fix_android_jni_static.patch dont_hardcode_pwd.patch -$(package)_patches += qtbase-moc-ignore-gcc-macro.patch fix_limits_header.patch -$(package)_patches += fix_bigsur_style.patch use_android_ndk23.patch +$(package)_patches = qt.pro +$(package)_patches += qttools_src.pro +$(package)_patches += mac-qmake.conf +$(package)_patches += fix_qt_pkgconfig.patch +$(package)_patches += no-xlib.patch +$(package)_patches += dont_hardcode_x86_64.patch +$(package)_patches += fix_montery_include.patch +$(package)_patches += fix_android_jni_static.patch +$(package)_patches += dont_hardcode_pwd.patch +$(package)_patches += qtbase-moc-ignore-gcc-macro.patch +$(package)_patches += fix_limits_header.patch +$(package)_patches += use_android_ndk23.patch $(package)_patches += rcc_hardcode_timestamp.patch +$(package)_patches += duplicate_lcqpafonts.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) -$(package)_qttranslations_sha256_hash=d5788e86257b21d5323f1efd94376a213e091d1e5e03b45a95dd052b5f570db8 +$(package)_qttranslations_sha256_hash=5d7869f670a135ad0986e266813b9dd5bbae2b09577338f9cdf8904d4af52db0 $(package)_qttools_file_name=qttools-$($(package)_suffix) -$(package)_qttools_sha256_hash=c189d0ce1ff7c739db9a3ace52ac3e24cb8fd6dbf234e49f075249b38f43c1cc +$(package)_qttools_sha256_hash=463b2fe71a085e7ab4e39333ae360ab0ec857b966d7a08f752c427e5df55f90d $(package)_extra_sources = $($(package)_qttranslations_file_name) $(package)_extra_sources += $($(package)_qttools_file_name) @@ -232,17 +239,15 @@ define $(package)_preprocess_cmds cp $($(package)_patch_dir)/qttools_src.pro qttools/src/src.pro && \ patch -p1 -i $($(package)_patch_dir)/dont_hardcode_pwd.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_qt_pkgconfig.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_no_printer.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_android_jni_static.patch && \ patch -p1 -i $($(package)_patch_dir)/no-xlib.patch && \ - patch -p1 -i $($(package)_patch_dir)/dont_use_avx_android_x86_64.patch && \ patch -p1 -i $($(package)_patch_dir)/dont_hardcode_x86_64.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_limits_header.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_montery_include.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_bigsur_style.patch && \ patch -p1 -i $($(package)_patch_dir)/use_android_ndk23.patch && \ patch -p1 -i $($(package)_patch_dir)/rcc_hardcode_timestamp.patch && \ + patch -p1 -i $($(package)_patch_dir)/duplicate_lcqpafonts.patch && \ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ cp -f qtbase/mkspecs/macx-clang/qplatformdefs.h qtbase/mkspecs/macx-clang-linux/ &&\ cp -f $($(package)_patch_dir)/mac-qmake.conf qtbase/mkspecs/macx-clang-linux/qmake.conf && \ diff --git a/depends/patches/native_libdmg-hfsplus/remove-libcrypto-dependency.patch b/depends/patches/native_libdmg-hfsplus/remove-libcrypto-dependency.patch deleted file mode 100644 index f346c8f2cf..0000000000 --- a/depends/patches/native_libdmg-hfsplus/remove-libcrypto-dependency.patch +++ /dev/null @@ -1,45 +0,0 @@ -From 3e5fd3fb56bc9ff03beb535979e33dcf83fe1f70 Mon Sep 17 00:00:00 2001 -From: Cory Fields <cory-nospam-@coryfields.com> -Date: Thu, 8 May 2014 12:39:42 -0400 -Subject: [PATCH] dmg: remove libcrypto dependency - ---- - dmg/CMakeLists.txt | 16 ---------------- - 1 file changed, 16 deletions(-) - -diff --git a/dmg/CMakeLists.txt b/dmg/CMakeLists.txt -index eec62d6..3969f64 100644 ---- a/dmg/CMakeLists.txt -+++ b/dmg/CMakeLists.txt -@@ -1,12 +1,5 @@ --INCLUDE(FindOpenSSL) - INCLUDE(FindZLIB) - --FIND_LIBRARY(CRYPTO_LIBRARIES crypto -- PATHS -- /usr/lib -- /usr/local/lib -- ) -- - IF(NOT ZLIB_FOUND) - message(FATAL_ERROR "zlib is required for dmg!") - ENDIF(NOT ZLIB_FOUND) -@@ -18,15 +11,6 @@ link_directories(${PROJECT_BINARY_DIR}/common ${PROJECT_BINARY_DIR}/hfs) - - add_library(dmg adc.c base64.c checksum.c dmgfile.c dmglib.c filevault.c io.c partition.c resources.c udif.c) - --IF(OPENSSL_FOUND) -- add_definitions(-DHAVE_CRYPT) -- include_directories(${OPENSSL_INCLUDE_DIR}) -- target_link_libraries(dmg ${CRYPTO_LIBRARIES}) -- IF(WIN32) -- TARGET_LINK_LIBRARIES(dmg gdi32) -- ENDIF(WIN32) --ENDIF(OPENSSL_FOUND) -- - target_link_libraries(dmg common hfs z) - - add_executable(dmg-bin dmg.c) --- -2.22.0 - diff --git a/depends/patches/qt/dont_use_avx_android_x86_64.patch b/depends/patches/qt/dont_use_avx_android_x86_64.patch deleted file mode 100644 index b12ac8f4d0..0000000000 --- a/depends/patches/qt/dont_use_avx_android_x86_64.patch +++ /dev/null @@ -1,32 +0,0 @@ -Android: don't use avx and avx2 when building for Android x86_64 - -AVX and AVX2 are not supported on x86_64 ABI. -See: - - https://developer.android.com/ndk/guides/abis#86-64 - - https://bugreports.qt.io/browse/QTBUG-86785 - -Upstream commits: - - Qt 6.0: ff1a44be33f4bc05d502a2ca49171e0408992f61 - - Qt 5.15: 8b2cc0f6deb038a4c9d4f0d9b690c7726bd809a9 - - ---- old/qtbase/configure.json -+++ new/qtbase/configure.json -@@ -1098,7 +1098,7 @@ - }, - "avx": { - "label": "AVX", -- "condition": "features.sse4_2 && tests.avx", -+ "condition": "features.sse4_2 && tests.avx && (!config.android || !arch.x86_64)", - "output": [ - "privateConfig", - { "type": "define", "name": "QT_COMPILER_SUPPORTS_AVX", "value": 1 } -@@ -1114,7 +1114,7 @@ - }, - "avx2": { - "label": "AVX2", -- "condition": "features.avx && tests.avx2", -+ "condition": "features.avx && tests.avx2 && (!config.android || !arch.x86_64)", - "output": [ - "privateConfig", - "privateFeature", diff --git a/depends/patches/qt/duplicate_lcqpafonts.patch b/depends/patches/qt/duplicate_lcqpafonts.patch new file mode 100644 index 0000000000..c460b51dcf --- /dev/null +++ b/depends/patches/qt/duplicate_lcqpafonts.patch @@ -0,0 +1,104 @@ +QtGui: Fix duplication of logging category lcQpaFonts + +Move it to qplatformfontdatabase.h. + +Upstream commit: + - Qt 6.0: ab01885e48873fb2ad71841a3f1627fe4d9cd835 + +--- a/qtbase/src/gui/text/qplatformfontdatabase.cpp ++++ b/qtbase/src/gui/text/qplatformfontdatabase.cpp +@@ -52,6 +52,8 @@ + + QT_BEGIN_NAMESPACE + ++Q_LOGGING_CATEGORY(lcQpaFonts, "qt.qpa.fonts") ++ + void qt_registerFont(const QString &familyname, const QString &stylename, + const QString &foundryname, int weight, + QFont::Style style, int stretch, bool antialiased, + +--- a/qtbase/src/gui/text/qplatformfontdatabase.h ++++ b/qtbase/src/gui/text/qplatformfontdatabase.h +@@ -50,6 +50,7 @@ + // + + #include <QtGui/qtguiglobal.h> ++#include <QtCore/qloggingcategory.h> + #include <QtCore/QString> + #include <QtCore/QStringList> + #include <QtCore/QList> +@@ -62,6 +63,7 @@ + + QT_BEGIN_NAMESPACE + ++Q_DECLARE_LOGGING_CATEGORY(lcQpaFonts) + + class QWritingSystemsPrivate; + + +--- a/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm ++++ b/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm +@@ -86,8 +86,6 @@ + + QT_BEGIN_NAMESPACE + +-Q_LOGGING_CATEGORY(lcQpaFonts, "qt.qpa.fonts") +- + static float SYNTHETIC_ITALIC_SKEW = std::tan(14.f * std::acos(0.f) / 90.f); + + bool QCoreTextFontEngine::ct_getSfntTable(void *user_data, uint tag, uchar *buffer, uint *length) + +--- a/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext_p.h ++++ b/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext_p.h +@@ -64,8 +64,6 @@ + + QT_BEGIN_NAMESPACE + +-Q_DECLARE_LOGGING_CATEGORY(lcQpaFonts) +- + class QCoreTextFontEngine : public QFontEngine + { + Q_GADGET + +--- a/qtbase/src/platformsupport/fontdatabases/windows/qwindowsfontdatabase.cpp ++++ b/qtbase/src/platformsupport/fontdatabases/windows/qwindowsfontdatabase.cpp +@@ -68,8 +68,6 @@ + + QT_BEGIN_NAMESPACE + +-Q_LOGGING_CATEGORY(lcQpaFonts, "qt.qpa.fonts") +- + #ifndef QT_NO_DIRECTWRITE + // ### fixme: Consider direct linking of dwrite.dll once Windows Vista pre SP2 is dropped (QTBUG-49711) + + +--- a/qtbase/src/platformsupport/fontdatabases/windows/qwindowsfontdatabase_p.h ++++ b/qtbase/src/platformsupport/fontdatabases/windows/qwindowsfontdatabase_p.h +@@ -63,8 +63,6 @@ + + QT_BEGIN_NAMESPACE + +-Q_DECLARE_LOGGING_CATEGORY(lcQpaFonts) +- + class QWindowsFontEngineData + { + Q_DISABLE_COPY_MOVE(QWindowsFontEngineData) + +--- a/qtbase/src/platformsupport/themes/genericunix/qgenericunixthemes.cpp ++++ b/qtbase/src/platformsupport/themes/genericunix/qgenericunixthemes.cpp +@@ -40,6 +40,7 @@ + #include "qgenericunixthemes_p.h" + + #include "qpa/qplatformtheme_p.h" ++#include "qpa/qplatformfontdatabase.h" + + #include <QtGui/QPalette> + #include <QtGui/QFont> +@@ -76,7 +77,6 @@ + QT_BEGIN_NAMESPACE + + Q_DECLARE_LOGGING_CATEGORY(qLcTray) +-Q_LOGGING_CATEGORY(lcQpaFonts, "qt.qpa.fonts") + + ResourceHelper::ResourceHelper() + { diff --git a/depends/patches/qt/fix_android_jni_static.patch b/depends/patches/qt/fix_android_jni_static.patch index bb64661761..22a4d5ab0e 100644 --- a/depends/patches/qt/fix_android_jni_static.patch +++ b/depends/patches/qt/fix_android_jni_static.patch @@ -1,6 +1,6 @@ --- old/qtbase/src/plugins/platforms/android/androidjnimain.cpp +++ new/qtbase/src/plugins/platforms/android/androidjnimain.cpp -@@ -914,6 +914,14 @@ Q_DECL_EXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void */*reserved*/) +@@ -934,6 +934,14 @@ Q_DECL_EXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void */*reserved*/) __android_log_print(ANDROID_LOG_FATAL, "Qt", "registerNatives failed"); return -1; } diff --git a/depends/patches/qt/fix_bigsur_style.patch b/depends/patches/qt/fix_bigsur_style.patch deleted file mode 100644 index 0f6c848bb0..0000000000 --- a/depends/patches/qt/fix_bigsur_style.patch +++ /dev/null @@ -1,90 +0,0 @@ -Fix rendering in macOS BigSur - -See: https://bugreports.qt.io/browse/QTBUG-86513. - -Upstream commits (combined in this patch): - - Qt 6.0: 40fb97e97f550b8afd13ecc3a038d9d0c2d82bbb - - Qt 6.0: 3857f104cac127f62e64e55a20613f0ac2e6b843 - - Qt 6.1: abee4cdd5925a8513f51784754fca8fa35031732 - ---- old/qtbase/src/plugins/styles/mac/qmacstyle_mac.mm -+++ new/qtbase/src/plugins/styles/mac/qmacstyle_mac.mm -@@ -3870,6 +3870,7 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter - const auto cs = d->effectiveAquaSizeConstrain(opt, w); - // Extra hacks to get the proper pressed appreance when not selected or selected and inactive - const bool needsInactiveHack = (!isActive && isSelected); -+ const bool isBigSurOrAbove = QOperatingSystemVersion::current() >= QOperatingSystemVersion::MacOSBigSur; - const auto ct = !needsInactiveHack && (isSelected || tp == QStyleOptionTab::OnlyOneTab) ? - QMacStylePrivate::Button_PushButton : - QMacStylePrivate::Button_PopupButton; -@@ -3878,6 +3879,12 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter - auto *pb = static_cast<NSButton *>(d->cocoaControl(cw)); - - auto vOffset = isPopupButton ? 1 : 2; -+ if (isBigSurOrAbove) { -+ // Make it 1, otherwise, offset is very visible compared -+ // to selected tab (which is not a popup button). -+ vOffset = 1; -+ } -+ - if (tabDirection == QMacStylePrivate::East) - vOffset -= 1; - const auto outerAdjust = isPopupButton ? 1 : 4; -@@ -3894,9 +3901,22 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter - frameRect = frameRect.adjusted(-innerAdjust, 0, outerAdjust, 0); - else - frameRect = frameRect.adjusted(-outerAdjust, 0, innerAdjust, 0); -+ -+ if (isSelected && isBigSurOrAbove) { -+ // 1 pixed of 'roundness' is still visible on the right -+ // (the left is OK, it's rounded). -+ frameRect = frameRect.adjusted(0, 0, 1, 0); -+ } -+ - break; - case QStyleOptionTab::Middle: - frameRect = frameRect.adjusted(-innerAdjust, 0, innerAdjust, 0); -+ -+ if (isSelected && isBigSurOrAbove) { -+ // 1 pixel of 'roundness' is still visible on both -+ // sides - left and right. -+ frameRect = frameRect.adjusted(-1, 0, 1, 0); -+ } - break; - case QStyleOptionTab::End: - // Pressed state hack: tweak adjustments in preparation for flip below -@@ -3904,6 +3924,11 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter - frameRect = frameRect.adjusted(-innerAdjust, 0, outerAdjust, 0); - else - frameRect = frameRect.adjusted(-outerAdjust, 0, innerAdjust, 0); -+ -+ if (isSelected && isBigSurOrAbove) { -+ // 1 pixel of 'roundness' is still visible on the left. -+ frameRect = frameRect.adjusted(-1, 0, 0, 0); -+ } - break; - case QStyleOptionTab::OnlyOneTab: - frameRect = frameRect.adjusted(-outerAdjust, 0, outerAdjust, 0); -@@ -3951,7 +3976,10 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter - NSPopUpArrowPosition oldPosition = NSPopUpArrowAtCenter; - NSPopUpButtonCell *pbCell = nil; - auto rAdjusted = r; -- if (isPopupButton && tp == QStyleOptionTab::OnlyOneTab) { -+ if (isPopupButton && (tp == QStyleOptionTab::OnlyOneTab || isBigSurOrAbove)) { -+ // Note: starting from macOS BigSur NSPopupButton has this -+ // arrow 'button' in a different place and it became -+ // quite visible 'in between' inactive tabs. - pbCell = static_cast<NSPopUpButtonCell *>(pb.cell); - oldPosition = pbCell.arrowPosition; - pbCell.arrowPosition = NSPopUpNoArrow; -@@ -3959,6 +3987,10 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter - // NSPopUpButton in this state is smaller. - rAdjusted.origin.x -= 3; - rAdjusted.size.width += 6; -+ if (isBigSurOrAbove) { -+ if (tp == QStyleOptionTab::End) -+ rAdjusted.origin.x -= 2; -+ } - } - } - diff --git a/depends/patches/qt/fix_limits_header.patch b/depends/patches/qt/fix_limits_header.patch index cb5a8cd1b5..258128c0ca 100644 --- a/depends/patches/qt/fix_limits_header.patch +++ b/depends/patches/qt/fix_limits_header.patch @@ -1,46 +1,9 @@ Fix compiling with GCC 11 -See: https://bugreports.qt.io/browse/QTBUG-90395. +Upstream: + - bug report: https://bugreports.qt.io/browse/QTBUG-89977 + - fix in Qt 6.1: 813a928c7c3cf98670b6043149880ed5c955efb9 -Upstream commits: - - Qt 5.15 -- unavailable as open source - - Qt 6.0: b2af6332ea37e45ab230a7a5d2d278f86d961b83 - - Qt 6.1: 9c56d4da2ff631a8c1c30475bd792f6c86bda53c - ---- old/qtbase/src/corelib/global/qendian.h -+++ new/qtbase/src/corelib/global/qendian.h -@@ -44,6 +44,8 @@ - #include <QtCore/qfloat16.h> - #include <QtCore/qglobal.h> - -+#include <limits> -+ - // include stdlib.h and hope that it defines __GLIBC__ for glibc-based systems - #include <stdlib.h> - #include <string.h> - ---- old/qtbase/src/corelib/global/qfloat16.h -+++ new/qtbase/src/corelib/global/qfloat16.h -@@ -43,6 +43,7 @@ - - #include <QtCore/qglobal.h> - #include <QtCore/qmetatype.h> -+#include <limits> - #include <string.h> - - #if defined(QT_COMPILER_SUPPORTS_F16C) && defined(__AVX2__) && !defined(__F16C__) - ---- old/qtbase/src/tools/moc/generator.cpp -+++ new/qtbase/src/tools/moc/generator.cpp -@@ -40,6 +40,8 @@ - #include <QtCore/qplugin.h> - #include <QtCore/qstringview.h> - -+#include <limits> -+ - #include <math.h> - #include <stdio.h> - --- old/qtbase/src/corelib/text/qbytearraymatcher.h +++ new/qtbase/src/corelib/text/qbytearraymatcher.h @@ -42,6 +42,8 @@ @@ -52,6 +15,12 @@ Upstream commits: QT_BEGIN_NAMESPACE + +Upstream fix and backports: + - Qt 6.1: 3eab20ad382569cb2c9e6ccec2322c3d08c0f716 + - Qt 6.2: 380294a5971da85010a708dc23b0edec192cbf27 + - Qt 6.3: 2b2b3155d9f6ba1e4f859741468fbc47db09292b + --- old/qtbase/src/corelib/tools/qoffsetstringarray_p.h +++ new/qtbase/src/corelib/tools/qoffsetstringarray_p.h @@ -55,6 +55,7 @@ diff --git a/depends/patches/qt/fix_no_printer.patch b/depends/patches/qt/fix_no_printer.patch deleted file mode 100644 index 1372356138..0000000000 --- a/depends/patches/qt/fix_no_printer.patch +++ /dev/null @@ -1,19 +0,0 @@ ---- x/qtbase/src/plugins/platforms/cocoa/qprintengine_mac_p.h -+++ y/qtbase/src/plugins/platforms/cocoa/qprintengine_mac_p.h -@@ -52,6 +52,7 @@ - // - - #include <QtCore/qglobal.h> -+#include <qpa/qplatformprintdevice.h> - - #ifndef QT_NO_PRINTER - ---- x/qtbase/src/plugins/plugins.pro -+++ y/qtbase/src/plugins/plugins.pro -@@ -9,6 +9,3 @@ qtHaveModule(gui) { - !android:qtConfig(library): SUBDIRS *= generic - } - qtHaveModule(widgets): SUBDIRS += styles -- --!winrt:qtHaveModule(printsupport): \ -- SUBDIRS += printsupport diff --git a/depends/patches/qt/mac-qmake.conf b/depends/patches/qt/mac-qmake.conf index e4bfaa1463..543407f853 100644 --- a/depends/patches/qt/mac-qmake.conf +++ b/depends/patches/qt/mac-qmake.conf @@ -19,6 +19,4 @@ QMAKE_MAC_SDK.macosx.PlatformPath = /phony !host_build: QMAKE_LFLAGS += -target $${MAC_TARGET} QMAKE_AR = $${CROSS_COMPILE}ar cq QMAKE_RANLIB=$${CROSS_COMPILE}ranlib -QMAKE_LIBTOOL=$${CROSS_COMPILE}libtool -QMAKE_INSTALL_NAME_TOOL=$${CROSS_COMPILE}install_name_tool load(qt_config) diff --git a/depends/patches/qt/use_android_ndk23.patch b/depends/patches/qt/use_android_ndk23.patch index 85b8690218..f22367d527 100644 --- a/depends/patches/qt/use_android_ndk23.patch +++ b/depends/patches/qt/use_android_ndk23.patch @@ -2,7 +2,7 @@ Use Android NDK r23 LTS --- old/qtbase/mkspecs/features/android/default_pre.prf +++ new/qtbase/mkspecs/features/android/default_pre.prf -@@ -73,7 +73,7 @@ else: equals(QT_ARCH, x86_64): CROSS_COMPILE = $$NDK_LLVM_PATH/bin/x86_64-linux- +@@ -76,7 +76,7 @@ else: equals(QT_ARCH, x86_64): CROSS_COMPILE = $$NDK_LLVM_PATH/bin/x86_64-linux- else: equals(QT_ARCH, arm64-v8a): CROSS_COMPILE = $$NDK_LLVM_PATH/bin/aarch64-linux-android- else: CROSS_COMPILE = $$NDK_LLVM_PATH/bin/arm-linux-androideabi- diff --git a/doc/README.md b/doc/README.md index c200ac3753..33f71f4807 100644 --- a/doc/README.md +++ b/doc/README.md @@ -73,6 +73,7 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th - [Assets Attribution](assets-attribution.md) - [Assumeutxo design](assumeutxo.md) - [bitcoin.conf Configuration File](bitcoin-conf.md) +- [CJDNS Support](cjdns.md) - [Files](files.md) - [Fuzz-testing](fuzzing.md) - [I2P Support](i2p.md) diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 1f0a07a284..4b46f29153 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -47,18 +47,24 @@ The HTTP request and response are both handled entirely in-memory. With the /notxdetails/ option JSON response will only contain the transaction hash instead of the complete transaction details. The option only affects the JSON response. #### Blockheaders -`GET /rest/headers/<COUNT>/<BLOCK-HASH>.<bin|hex|json>` +`GET /rest/headers/<BLOCK-HASH>.<bin|hex|json>?count=<COUNT=5>` Given a block hash: returns <COUNT> amount of blockheaders in upward direction. Returns empty if the block doesn't exist or it isn't in the active chain. +*Deprecated (but not removed) since 24.0:* +`GET /rest/headers/<COUNT>/<BLOCK-HASH>.<bin|hex|json>` + #### Blockfilter Headers -`GET /rest/blockfilterheaders/<FILTERTYPE>/<COUNT>/<BLOCK-HASH>.<bin|hex|json>` +`GET /rest/blockfilterheaders/<FILTERTYPE>/<BLOCK-HASH>.<bin|hex|json>?count=<COUNT=5>` Given a block hash: returns <COUNT> amount of blockfilter headers in upward direction for the filter type <FILTERTYPE>. Returns empty if the block doesn't exist or it isn't in the active chain. +*Deprecated (but not removed) since 24.0:* +`GET /rest/blockfilterheaders/<FILTERTYPE>/<COUNT>/<BLOCK-HASH>.<bin|hex|json>` + #### Blockfilters `GET /rest/blockfilter/<FILTERTYPE>/<BLOCK-HASH>.<bin|hex|json>` @@ -76,17 +82,7 @@ Given a height: returns hash of block in best-block-chain at height provided. Returns various state info regarding block chain processing. Only supports JSON as output format. -* chain : (string) current network name (main, test, signet, regtest) -* blocks : (numeric) the current number of blocks processed in the server -* headers : (numeric) the current number of headers we have validated -* bestblockhash : (string) the hash of the currently best block -* difficulty : (numeric) the current difficulty -* mediantime : (numeric) the median time of the 11 blocks before the most recent block on the blockchain -* verificationprogress : (numeric) estimate of verification progress [0..1] -* chainwork : (string) total amount of work in active chain, in hexadecimal -* pruned : (boolean) if the blocks are subject to pruning -* pruneheight : (numeric) highest block available -* softforks : (array) status of softforks in progress +Refer to the `getblockchaininfo` RPC help for details. #### Query UTXO set `GET /rest/getutxos/<checkmempool>/<txid>-<n>/<txid>-<n>/.../<txid>-<n>.<bin|hex|json>` @@ -121,14 +117,15 @@ $ curl localhost:18332/rest/getutxos/checkmempool/b2cdfd7b89def827ff8af7cd9bff76 #### Memory pool `GET /rest/mempool/info.json` -Returns various information about the TX mempool. +Returns various information about the transaction mempool. Only supports JSON as output format. -Refer to the `getmempoolinfo` RPC for documentation of the fields. +Refer to the `getmempoolinfo` RPC help for details. `GET /rest/mempool/contents.json` -Returns transactions in the TX mempool. +Returns the transactions in the mempool. Only supports JSON as output format. +Refer to the `getrawmempool` RPC help for details. Risks ------------- diff --git a/doc/bitcoin-conf.md b/doc/bitcoin-conf.md index 8c9035c45b..acd767f234 100644 --- a/doc/bitcoin-conf.md +++ b/doc/bitcoin-conf.md @@ -61,4 +61,12 @@ Windows | `%APPDATA%\Bitcoin\` | `C:\Users\username\AppData\Roaming\Bitcoin\bitc Linux | `$HOME/.bitcoin/` | `/home/username/.bitcoin/bitcoin.conf` macOS | `$HOME/Library/Application Support/Bitcoin/` | `/Users/username/Library/Application Support/Bitcoin/bitcoin.conf` -You can find an example bitcoin.conf file in [share/examples/bitcoin.conf](../share/examples/bitcoin.conf). +An example configuration file can be generated by [contrib/devtools/gen-bitcoin-conf.sh](../contrib/devtools/gen-bitcoin-conf.sh). +Run this script after compiling to generate an up-to-date configuration file. +The output is placed under `share/examples/bitcoin.conf`. +To use the generated configuration file, copy the example file into your data directory and edit it there, like so: + +``` +# example copy command for linux user +cp share/examples/bitcoin.conf ~/.bitcoin +``` diff --git a/doc/build-freebsd.md b/doc/build-freebsd.md index da2ab61c2a..a8e643a2ab 100644 --- a/doc/build-freebsd.md +++ b/doc/build-freebsd.md @@ -1,48 +1,21 @@ # FreeBSD Build Guide -**Updated for FreeBSD [12.2](https://www.freebsd.org/releases/12.2R/announce.html)** +**Updated for FreeBSD [12.3](https://www.freebsd.org/releases/12.3R/announce/)** This guide describes how to build bitcoind, command-line utilities, and GUI on FreeBSD. -## Dependencies - -The following dependencies are **required**: - - Library | Purpose | Description - ----------------------------------------------------------------------|------------|---------------------- - [autoconf](https://svnweb.freebsd.org/ports/head/devel/autoconf/) | Build | Automatically configure software source code - [automake](https://svnweb.freebsd.org/ports/head/devel/automake/) | Build | Generate makefile (requires autoconf) - [libtool](https://svnweb.freebsd.org/ports/head/devel/libtool/) | Build | Shared library support - [pkgconf](https://svnweb.freebsd.org/ports/head/devel/pkgconf/) | Build | Configure compiler and linker flags - [git](https://svnweb.freebsd.org/ports/head/devel/git/) | Clone | Version control system - [gmake](https://svnweb.freebsd.org/ports/head/devel/gmake/) | Compile | Generate executables - [boost-libs](https://svnweb.freebsd.org/ports/head/devel/boost-libs/) | Utility | Library for threading, data structures, etc - [libevent](https://svnweb.freebsd.org/ports/head/devel/libevent/) | Networking | OS independent asynchronous networking - - -The following dependencies are **optional**: - - Library | Purpose | Description - ---------------------------------------------------------------------------|------------------|---------------------- - [db5](https://svnweb.freebsd.org/ports/head/databases/db5/) | Berkeley DB | Wallet storage (only needed when wallet enabled) - [qt5](https://svnweb.freebsd.org/ports/head/devel/qt5/) | GUI | GUI toolkit (only needed when GUI enabled) - [libqrencode](https://svnweb.freebsd.org/ports/head/graphics/libqrencode/) | QR codes in GUI | Generating QR codes (only needed when GUI enabled) - [libzmq4](https://svnweb.freebsd.org/ports/head/net/libzmq4/) | ZMQ notification | Allows generating ZMQ notifications (requires ZMQ version >= 4.0.0) - [sqlite3](https://svnweb.freebsd.org/ports/head/databases/sqlite3/) | SQLite DB | Wallet storage (only needed when wallet enabled) - [python3](https://svnweb.freebsd.org/ports/head/lang/python3/) | Testing | Python Interpreter (only needed when running the test suite) - - See [dependencies.md](dependencies.md) for a complete overview. - ## Preparation ### 1. Install Required Dependencies -Install the required dependencies the usual way you [install software on FreeBSD](https://www.freebsd.org/doc/en/books/handbook/ports.html) - either with `pkg` or via the Ports collection. The example commands below use `pkg` which is usually run as `root` or via `sudo`. If you want to use `sudo`, and you haven't set it up: [use this guide](http://www.freebsdwiki.net/index.php/Sudo%2C_configuring) to setup `sudo` access on FreeBSD. +Run the following as root to install the base dependencies for building. ```bash pkg install autoconf automake boost-libs git gmake libevent libtool pkgconf ``` +See [dependencies.md](dependencies.md) for a complete overview. + ### 2. Clone Bitcoin Repo Now that `git` and all the required dependencies are installed, let's clone the Bitcoin Core repository to a directory. All build scripts and commands will run from this directory. ``` bash @@ -52,21 +25,23 @@ git clone https://github.com/bitcoin/bitcoin.git ### 3. Install Optional Dependencies #### Wallet Dependencies -It is not necessary to build wallet functionality to run bitcoind or the GUI. To enable legacy wallets, you must install `db5`. To enable [descriptor wallets](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md), `sqlite3` is required. Skip `db5` if you intend to *exclusively* use descriptor wallets - -###### Legacy Wallet Support -`db5` is required to enable support for legacy wallets. Skip if you don't intend to use legacy wallets - -```bash -pkg install db5 -``` +It is not necessary to build wallet functionality to run either `bitcoind` or `bitcoin-qt`. ###### Descriptor Wallet Support -`sqlite3` is required to enable support for descriptor wallets. Skip if you don't intend to use descriptor wallets. +`sqlite3` is required to support [descriptor wallets](descriptors.md). +Skip if you don't intend to use descriptor wallets. ``` bash pkg install sqlite3 ``` + +###### Legacy Wallet Support +`db5` is only required to support legacy wallets. +Skip if you don't intend to use legacy wallets. + +```bash +pkg install db5 +``` --- #### GUI Dependencies @@ -84,6 +59,14 @@ pkg install libqrencode ``` --- +#### Notifications +###### ZeroMQ + +Bitcoin Core can provide notifications via ZeroMQ. If the package is installed, support will be compiled in. +```bash +pkg install libzmq4 +``` + #### Test Suite Dependencies There is an included test suite that is useful for testing code changes when developing. To run the test suite (recommended), you will need to have Python 3 installed: @@ -98,8 +81,17 @@ pkg install python3 ### 1. Configuration There are many ways to configure Bitcoin Core, here are a few common examples: -##### Wallet (BDB + SQlite) Support, No GUI: -This explicitly enables legacy wallet support and disables the GUI. If `sqlite3` is installed, then descriptor wallet support will be built. + +##### Descriptor Wallet and GUI: +This explicitly enables the GUI and disables legacy wallet support, assuming `sqlite` and `qt` are installed. +```bash +./autogen.sh +./configure --without-bdb --with-gui=yes MAKE=gmake +``` + +##### Descriptor & Legacy Wallet. No GUI: +This enables support for both wallet types and disables the GUI, assuming +`sqlite3` and `db5` are both installed. ```bash ./autogen.sh ./configure --with-gui=no --with-incompatible-bdb \ @@ -108,12 +100,6 @@ This explicitly enables legacy wallet support and disables the GUI. If `sqlite3` MAKE=gmake ``` -##### Wallet (only SQlite) and GUI Support: -This explicitly enables the GUI and disables legacy wallet support. If `qt5` is not installed, this will throw an error. If `sqlite3` is installed then descriptor wallet functionality will be built. If `sqlite3` is not installed, then wallet functionality will be disabled. -```bash -./autogen.sh -./configure --without-bdb --with-gui=yes MAKE=gmake -``` ##### No Wallet or GUI ``` bash ./autogen.sh diff --git a/doc/build-netbsd.md b/doc/build-netbsd.md index edabd71611..ba9a80f83b 100644 --- a/doc/build-netbsd.md +++ b/doc/build-netbsd.md @@ -27,15 +27,33 @@ git clone https://github.com/bitcoin/bitcoin.git See [dependencies.md](dependencies.md) for a complete overview. -### Building BerkeleyDB +### Building Bitcoin Core + +**Important**: Use `gmake` (the non-GNU `make` will exit with an error). + +#### With descriptor wallet: + +The descriptor wallet uses `sqlite3`. You can install it using: +```bash +pkgin install sqlite3 +``` + +```bash +./autogen.sh +./configure --with-gui=no --without-bdb \ + CPPFLAGS="-I/usr/pkg/include" \ + LDFLAGS="-L/usr/pkg/lib" \ + BOOST_CPPFLAGS="-I/usr/pkg/include" \ + MAKE=gmake +``` -BerkeleyDB is only necessary for the wallet functionality. To skip this, pass -`--disable-wallet` to `./configure` and skip to the next section. +#### With legacy wallet: + +BerkeleyDB is use for legacy wallet functionality. It is recommended to use Berkeley DB 4.8. You cannot use the BerkeleyDB library -from ports, for the same reason as boost above (g++/libstd++ incompatibility). -If you have to build it yourself, you can use [the installation script included -in contrib/](/contrib/install_db4.sh) like so: +from ports. +You can use [the installation script included in contrib/](/contrib/install_db4.sh) like so: ```bash ./contrib/install_db4.sh `pwd` @@ -47,30 +65,23 @@ from the root of the repository. Then set `BDB_PREFIX` for the next section: export BDB_PREFIX="$PWD/db4" ``` -### Building Bitcoin Core - -**Important**: Use `gmake` (the non-GNU `make` will exit with an error). - -With wallet: ```bash ./autogen.sh ./configure --with-gui=no CPPFLAGS="-I/usr/pkg/include" \ LDFLAGS="-L/usr/pkg/lib" \ BOOST_CPPFLAGS="-I/usr/pkg/include" \ - BOOST_LDFLAGS="-L/usr/pkg/lib" \ BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ BDB_CFLAGS="-I${BDB_PREFIX}/include" \ MAKE=gmake ``` -Without wallet: +#### Without wallet: ```bash ./autogen.sh ./configure --with-gui=no --disable-wallet \ CPPFLAGS="-I/usr/pkg/include" \ LDFLAGS="-L/usr/pkg/lib" \ BOOST_CPPFLAGS="-I/usr/pkg/include" \ - BOOST_LDFLAGS="-L/usr/pkg/lib" \ MAKE=gmake ``` diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index 275b7ce124..99daf73c86 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -1,123 +1,133 @@ -OpenBSD build guide -====================== -(updated for OpenBSD 6.9) +# OpenBSD Build Guide -This guide describes how to build bitcoind, bitcoin-qt, and command-line utilities on OpenBSD. +**Updated for OpenBSD [7.0](https://www.openbsd.org/70.html)** -Preparation -------------- +This guide describes how to build bitcoind, command-line utilities, and GUI on OpenBSD. -Run the following as root to install the base dependencies for building: +## Preparation + +### 1. Install Required Dependencies +Run the following as root to install the base dependencies for building. ```bash -pkg_add git gmake libevent libtool boost -pkg_add qt5 # (optional for enabling the GUI) -pkg_add autoconf # (select highest version, e.g. 2.69) -pkg_add automake # (select highest version, e.g. 1.16) -pkg_add python # (select highest version, e.g. 3.8) -pkg_add bash +pkg_add bash git gmake libevent libtool boost +# Select the newest version of the following packages: +pkg_add autoconf automake python +``` +See [dependencies.md](dependencies.md) for a complete overview. + +### 2. Clone Bitcoin Repo +Clone the Bitcoin Core repository to a directory. All build scripts and commands will run from this directory. +``` bash git clone https://github.com/bitcoin/bitcoin.git ``` -See [dependencies.md](dependencies.md) for a complete overview. +### 3. Install Optional Dependencies -**Important**: From OpenBSD 6.2 onwards a C++11-supporting clang compiler is -part of the base image, and while building it is necessary to make sure that -this compiler is used and not ancient g++ 4.2.1. This is done by appending -`CC=cc CXX=c++` to configuration commands. Mixing different compilers within -the same executable will result in errors. +#### Wallet Dependencies -### Building BerkeleyDB +It is not necessary to build wallet functionality to run either `bitcoind` or `bitcoin-qt`. + +###### Descriptor Wallet Support + +`sqlite3` is required to support [descriptor wallets](descriptors.md). + +``` bash +pkg_add install sqlite3 +``` -BerkeleyDB is only necessary for the wallet functionality. To skip this, pass -`--disable-wallet` to `./configure` and skip to the next section. +###### Legacy Wallet Support +BerkeleyDB is only required to support legacy wallets. It is recommended to use Berkeley DB 4.8. You cannot use the BerkeleyDB library -from ports, for the same reason as boost above (g++/libstd++ incompatibility). -If you have to build it yourself, you can use [the installation script included -in contrib/](/contrib/install_db4.sh) like so: +from ports. However you can build it yourself, [using the installation script included in contrib/](/contrib/install_db4.sh), like so, from the root of the repository. ```bash -./contrib/install_db4.sh `pwd` CC=cc CXX=c++ +./contrib/install_db4.sh `pwd` ``` -from the root of the repository. Then set `BDB_PREFIX` for the next section: +Then set `BDB_PREFIX`: ```bash export BDB_PREFIX="$PWD/db4" ``` -### Building Bitcoin Core +#### GUI Dependencies +###### Qt5 + +Bitcoin Core includes a GUI built with the cross-platform Qt Framework. To compile the GUI, Qt 5 is required. + +```bash +pkg_add qt5 +``` + +## Building Bitcoin Core **Important**: Use `gmake` (the non-GNU `make` will exit with an error). Preparation: ```bash -# Replace this with the autoconf version that you installed. Include only -# the major and minor parts of the version: use "2.69" for "autoconf-2.69p2". -export AUTOCONF_VERSION=2.69 - -# Replace this with the automake version that you installed. Include only -# the major and minor parts of the version: use "1.16" for "automake-1.16.1". +# Adapt the following for the version you installed (major.minor only): +export AUTOCONF_VERSION=2.71 export AUTOMAKE_VERSION=1.16 ./autogen.sh ``` -Make sure `BDB_PREFIX` is set to the appropriate path from the above steps. + +### 1. Configuration Note that building with external signer support currently fails on OpenBSD, hence you have to explicitly disable it by passing the parameter -`--disable-external-signer` to the configure script. -(Background: the feature requires the header-only library boost::process, which -is available on OpenBSD 6.9 via Boost 1.72.0, but contains certain system calls -and preprocessor defines like `waitid()` and `WEXITED` that are not available.) +`--disable-external-signer` to the configure script. The feature requires the +header-only library boost::process, which is available on OpenBSD, but contains +certain system calls and preprocessor defines like `waitid()` and `WEXITED` that +are not available. -To configure with wallet: -```bash -./configure --with-gui=no --disable-external-signer CC=cc CXX=c++ \ - BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ - BDB_CFLAGS="-I${BDB_PREFIX}/include" \ - MAKE=gmake -``` +There are many ways to configure Bitcoin Core, here are a few common examples: + +##### Descriptor Wallet and GUI: +This enables the GUI and descriptor wallet support, assuming `sqlite` and `qt5` are installed. -To configure without wallet: ```bash -./configure --disable-wallet --with-gui=no --disable-external-signer CC=cc CXX=c++ MAKE=gmake +./configure --disable-external-signer MAKE=gmake ``` -To configure with GUI: +##### Descriptor & Legacy Wallet. No GUI: +This enables support for both wallet types and disables the GUI: + ```bash -./configure --with-gui=yes --disable-external-signer CC=cc CXX=c++ \ +./configure --disable-external-signer --with-gui=no \ BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ BDB_CFLAGS="-I${BDB_PREFIX}/include" \ MAKE=gmake ``` -Build and run the tests: +### 2. Compile +**Important**: Use `gmake` (the non-GNU `make` will exit with an error). + ```bash -gmake # use "-j N" here for N parallel jobs -gmake check +gmake # use "-j N" for N parallel jobs +gmake check # Run tests if Python 3 is available ``` -Resource limits -------------------- +## Resource limits If the build runs into out-of-memory errors, the instructions in this section might help. The standard ulimit restrictions in OpenBSD are very strict: - - data(kbytes) 1572864 +```bash +data(kbytes) 1572864 +``` This is, unfortunately, in some cases not enough to compile some `.cpp` files in the project, (see issue [#6658](https://github.com/bitcoin/bitcoin/issues/6658)). If your user is in the `staff` group the limit can be raised with: - - ulimit -d 3000000 - +```bash +ulimit -d 3000000 +``` The change will only affect the current shell and processes spawned by it. To make the change system-wide, change `datasize-cur` and `datasize-max` in `/etc/login.conf`, and reboot. - diff --git a/doc/build-osx.md b/doc/build-osx.md index 16dc224aed..fdf0a9d414 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -4,42 +4,6 @@ This guide describes how to build bitcoind, command-line utilities, and GUI on macOS -**Note:** The following is for Intel Macs only! - -## Dependencies - -The following dependencies are **required**: - -Library | Purpose | Description ------------------------------------------------------------|------------|---------------------- -[automake](https://formulae.brew.sh/formula/automake) | Build | Generate makefile -[libtool](https://formulae.brew.sh/formula/libtool) | Build | Shared library support -[pkg-config](https://formulae.brew.sh/formula/pkg-config) | Build | Configure compiler and linker flags -[boost](https://formulae.brew.sh/formula/boost) | Utility | Library for threading, data structures, etc -[libevent](https://formulae.brew.sh/formula/libevent) | Networking | OS independent asynchronous networking - -The following dependencies are **optional**: - -Library | Purpose | Description ---------------------------------------------------------------- |------------------|---------------------- -[berkeley-db@4](https://formulae.brew.sh/formula/berkeley-db@4) | Berkeley DB | Wallet storage (only needed when wallet enabled) -[qt@5](https://formulae.brew.sh/formula/qt@5) | GUI | GUI toolkit (only needed when GUI enabled) -[qrencode](https://formulae.brew.sh/formula/qrencode) | QR codes in GUI | Generating QR codes (only needed when GUI enabled) -[zeromq](https://formulae.brew.sh/formula/zeromq) | ZMQ notification | Allows generating ZMQ notifications (requires ZMQ version >= 4.0.0) -[sqlite](https://formulae.brew.sh/formula/sqlite) | SQLite DB | Wallet storage (only needed when wallet enabled) -[miniupnpc](https://formulae.brew.sh/formula/miniupnpc) | UPnP Support | Firewall-jumping support (needed for port mapping support) -[libnatpmp](https://formulae.brew.sh/formula/libnatpmp) | NAT-PMP Support | Firewall-jumping support (needed for port mapping support) -[python3](https://formulae.brew.sh/formula/python@3.9) | Testing | Python Interpreter (only needed when running the test suite) - -The following dependencies are **optional** packages required for deploying: - -Library | Purpose | Description -----------------------------------------------------|------------------|---------------------- -[ds_store](https://pypi.org/project/ds-store/) | Deploy Dependency| Examine and modify .DS_Store files -[mac_alias](https://pypi.org/project/mac-alias/) | Deploy Dependency| Generate/Read binary alias and bookmark records - -See [dependencies.md](dependencies.md) for a complete overview. - ## Preparation The commands in this guide should be executed in a Terminal application. @@ -78,6 +42,9 @@ Note: If you run into issues while installing Homebrew or pulling packages, refe The first step is to download the required dependencies. These dependencies represent the packages required to get a barebones installation up and running. + +See [dependencies.md](dependencies.md) for a complete overview. + To install, run the following from your terminal: ``` bash @@ -99,29 +66,21 @@ git clone https://github.com/bitcoin/bitcoin.git #### Wallet Dependencies It is not necessary to build wallet functionality to run `bitcoind` or `bitcoin-qt`. -To enable legacy wallets, you must install `berkeley-db@4`. -To enable [descriptor wallets](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md), `sqlite` is required. -Skip `berkeley-db@4` if you intend to *exclusively* use descriptor wallets. - -###### Legacy Wallet Support -`berkeley-db@4` is required to enable support for legacy wallets. -Skip if you don't intend to use legacy wallets. +###### Descriptor Wallet Support -``` bash -brew install berkeley-db@4 -``` +`sqlite` is required to support for descriptor wallets. -###### Descriptor Wallet Support +macOS ships with a useable `sqlite` package, meaning you don't need to +install anything. -Note: Apple has included a useable `sqlite` package since macOS 10.14. -You may not need to install this package. +###### Legacy Wallet Support -`sqlite` is required to enable support for descriptor wallets. -Skip if you don't intend to use descriptor wallets. +`berkeley-db@4` is only required to support for legacy wallets. +Skip if you don't intend to use legacy wallets. ``` bash -brew install sqlite +brew install berkeley-db@4 ``` --- diff --git a/doc/build-unix.md b/doc/build-unix.md index 15fe63d047..f02729ee32 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -26,30 +26,7 @@ make install # optional This will build bitcoin-qt as well, if the dependencies are met. -Dependencies ---------------------- - -These dependencies are required: - - Library | Purpose | Description - ------------|------------------|---------------------- - libboost | Utility | Library for threading, data structures, etc - libevent | Networking | OS independent asynchronous networking - -Optional dependencies: - - Library | Purpose | Description - ------------|------------------|---------------------- - miniupnpc | UPnP Support | Firewall-jumping support - libnatpmp | NAT-PMP Support | Firewall-jumping support - libdb4.8 | Berkeley DB | Wallet storage (only needed when legacy wallet enabled) - qt | GUI | GUI toolkit (only needed when GUI enabled) - libqrencode | QR codes in GUI | QR code generation (only needed when GUI enabled) - libzmq3 | ZMQ notification | ZMQ notifications (requires ZMQ version >= 4.0.0) - sqlite3 | SQLite DB | Wallet storage (only needed when descriptor wallet enabled) - systemtap | Tracing (USDT) | Statically defined tracepoints - -For the versions used, see [dependencies.md](dependencies.md) +See [dependencies.md](dependencies.md) for a complete overview. Memory Requirements -------------------- @@ -232,7 +209,7 @@ from the root of the repository. Otherwise, you can build Bitcoin Core from self-compiled [depends](/depends/README.md). -**Note**: You only need Berkeley DB if the wallet is enabled (see [*Disable-wallet mode*](#disable-wallet-mode)). +**Note**: You only need Berkeley DB if the legacy wallet is enabled (see [*Disable-wallet mode*](#disable-wallet-mode)). Security -------- @@ -282,12 +259,12 @@ Hardening enables the following features: Disable-wallet mode -------------------- -When the intention is to run only a P2P node without a wallet, Bitcoin Core may be compiled in -disable-wallet mode with: +When the intention is to only run a P2P node, without a wallet, Bitcoin Core can +be compiled in disable-wallet mode with: ./configure --disable-wallet -In this case there is no dependency on Berkeley DB 4.8 and SQLite. +In this case there is no dependency on SQLite or Berkeley DB. Mining is also possible in disable-wallet mode using the `getblocktemplate` RPC call. diff --git a/doc/cjdns.md b/doc/cjdns.md new file mode 100644 index 0000000000..b69564729f --- /dev/null +++ b/doc/cjdns.md @@ -0,0 +1,116 @@ +# CJDNS support in Bitcoin Core + +It is possible to run Bitcoin Core over CJDNS, an encrypted IPv6 network that +uses public-key cryptography for address allocation and a distributed hash table +for routing. + +## What is CJDNS? + +CJDNS is like a distributed, shared VPN with multiple entry points where every +participant can reach any other participant. All participants use addresses from +the `fc00::/8` network (reserved IPv6 range). Installation and configuration is +done outside of Bitcoin Core, similarly to a VPN (either in the host/OS or on +the network router). See https://github.com/cjdelisle/cjdns#readme and +https://github.com/hyperboria/docs#hyperboriadocs for more information. + +Compared to IPv4/IPv6, CJDNS provides end-to-end encryption and protects nodes +from traffic analysis and filtering. + +Used with Tor and I2P, CJDNS is a complementary option that can enhance network +redundancy and robustness for both the Bitcoin network and individual nodes. + +Each network has different characteristics. For instance, Tor is widely used but +somewhat centralized. I2P connections have a source address and I2P is slow. +CJDNS is fast but does not hide the sender and the recipient from intermediate +routers. + +## Installing CJDNS and finding a peer to connect to the network + +To install and set up CJDNS, follow the instructions at +https://github.com/cjdelisle/cjdns#how-to-install-cjdns. + +You need to initiate an outbound connection to a peer on the CJDNS network +before it will work with your Bitcoin Core node. This is described in steps +["2. Find a friend"](https://github.com/cjdelisle/cjdns#2-find-a-friend) and +["3. Connect your node to your friend's +node"](https://github.com/cjdelisle/cjdns#3-connect-your-node-to-your-friends-node) +in the CJDNS documentation. + +One quick way to accomplish these two steps is to query for available public +peers on [Hyperboria](https://github.com/hyperboria) by running the following: + +``` +git clone https://github.com/hyperboria/peers hyperboria-peers +cd hyperboria-peers +./testAvailable.py +``` + +For each peer, the `./testAvailable.py` script prints the filename of the peer's +credentials followed by the ping result. + +Choose one or several peers, copy their credentials from their respective files, +paste them into the relevant IPv4 or IPv6 "connectTo" JSON object in the +`cjdroute.conf` file you created in step ["1. Generate a new configuration +file"](https://github.com/cjdelisle/cjdns#1-generate-a-new-configuration-file), +and save the file. + +## Launching CJDNS + +Typically, CJDNS might be launched from its directory with +`sudo ./cjdroute < cjdroute.conf` and it sheds permissions after setting up the +[TUN](https://en.wikipedia.org/wiki/TUN/TAP) interface. You may also [launch it as an +unprivileged user](https://github.com/cjdelisle/cjdns/blob/master/doc/non-root-user.md) +with some additional setup. + +The network connection can be checked by running `./tools/peerStats` from the +CJDNS directory. + +## Run Bitcoin Core with CJDNS + +Once you are connected to the CJDNS network, the following Bitcoin Core +configuration option makes CJDNS peers automatically reachable: + +``` +-cjdnsreachable +``` + +When enabled, this option tells Bitcoin Core that it is running in an +environment where a connection to an `fc00::/8` address will be to the CJDNS +network instead of to an [RFC4193](https://datatracker.ietf.org/doc/html/rfc4193) +IPv6 local network. This helps Bitcoin Core perform better address management: + - Your node can consider incoming `fc00::/8` connections to be from the CJDNS + network rather than from an IPv6 private one. + - If one of your node's local addresses is `fc00::/8`, then it can choose to + gossip that address to peers. + +## Additional configuration options related to CJDNS + +``` +-onlynet=cjdns +``` + +Make automatic outbound connections only to CJDNS addresses. Inbound and manual +connections are not affected by this option. It can be specified multiple times +to allow multiple networks, e.g. onlynet=cjdns, onlynet=i2p, onlynet=onion. + +CJDNS support was added to Bitcoin Core in version 23.0 and there may be fewer +CJDNS peers than Tor or IP ones. You can use `bitcoin-cli -addrinfo` to see the +number of CJDNS addresses known to your node. + +In general, a node can be run with both an onion service and CJDNS (or any/all +of IPv4/IPv6/onion/I2P/CJDNS), which can provide a potential fallback if one of +the networks has issues. There are a number of ways to configure this; see +[doc/tor.md](https://github.com/bitcoin/bitcoin/blob/master/doc/tor.md) for +details. + +## CJDNS-related information in Bitcoin Core + +There are several ways to see your CJDNS address in Bitcoin Core: +- in the "Local addresses" output of CLI `-netinfo` +- in the "localaddresses" output of RPC `getnetworkinfo` + +To see which CJDNS peers your node is connected to, use `bitcoin-cli -netinfo 4` +or the `getpeerinfo` RPC (i.e. `bitcoin-cli getpeerinfo`). + +To see which CJDNS addresses your node knows, use the `getnodeaddresses 0 cjdns` +RPC. diff --git a/doc/dependencies.md b/doc/dependencies.md index 99ff26f457..697d432520 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -1,48 +1,50 @@ -Dependencies -============ - -These are the dependencies currently used by Bitcoin Core. You can find instructions for installing them in the `build-*.md` file for your platform. - -| Dependency | Version used | Minimum required | CVEs | Shared | [Bundled Qt library](https://doc.qt.io/qt-5/configure-options.html#third-party-libraries) | -| --- | --- | --- | --- | --- | --- | -| Berkeley DB | [4.8.30](https://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html) | 4.8.x | No | | | -| Boost | [1.77.0](https://www.boost.org/users/download/) | [1.64.0](https://github.com/bitcoin/bitcoin/pull/22320) | No | | | -| Clang<sup>[ \* ](#note1)</sup> | | [7.0](https://releases.llvm.org/download.html) (C++17 & std::filesystem support) | | | | -| Fontconfig | [2.12.6](https://www.freedesktop.org/software/fontconfig/release/) | | No | Yes | | -| FreeType | [2.11.0](https://download.savannah.gnu.org/releases/freetype) | | No | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Android only) | -| GCC | | [8.1](https://gcc.gnu.org/) (C++17 & std::filesystem support) | | | | -| glibc | | [2.18](https://www.gnu.org/software/libc/) | | | | | -| HarfBuzz-NG | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | -| libevent | [2.1.12-stable](https://github.com/libevent/libevent/releases) | [2.0.21](https://github.com/bitcoin/bitcoin/pull/18676) | No | | | -| libnatpmp | git commit [4536032...](https://github.com/miniupnp/libnatpmp/tree/4536032ae32268a45c073a4d5e91bbab4534773a) | | No | | | -| libpng | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | -| MiniUPnPc | [2.2.2](https://miniupnp.tuxfamily.org/files) | | No | | | -| PCRE | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | -| Python (tests) | | [3.6](https://www.python.org/downloads) | | | | -| qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | | -| Qt | [5.15.2](https://download.qt.io/official_releases/qt/) | [5.9.5](https://github.com/bitcoin/bitcoin/issues/20104) | No | | | -| SQLite | [3.32.1](https://sqlite.org/download.html) | [3.7.17](https://github.com/bitcoin/bitcoin/pull/19077) | | | | -| XCB | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | -| systemtap ([tracing](tracing.md))| [4.5](https://sourceware.org/systemtap/ftp/releases/) | | | | | -| xkbcommon | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | -| ZeroMQ | [4.3.1](https://github.com/zeromq/libzmq/releases) | 4.0.0 | No | | | -| zlib | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | - -<a name="note1">Note \*</a> : When compiling with `-stdlib=libc++`, the minimum supported libc++ version is 7.0. - -Controlling dependencies ------------------------- -Some dependencies are not needed in all configurations. The following are some factors that affect the dependency list. - -#### Options passed to `./configure` -* MiniUPnPc is not needed with `--without-miniupnpc`. -* libnatpmp is not needed with `--without-natpmp`. -* Berkeley DB is not needed with `--disable-wallet` or `--without-bdb`. -* SQLite is not needed with `--disable-wallet` or `--without-sqlite`. -* Qt is not needed with `--without-gui`. -* If the qrencode dependency is absent, QR support won't be added. To force an error when that happens, pass `--with-qrencode`. -* If the systemtap dependency is absent, USDT support won't compiled in. -* ZeroMQ is needed only with the `--with-zmq` option. - -#### Other -* Not-Qt-bundled zlib is required to build the [DMG tool](../contrib/macdeploy/README.md#deterministic-macos-dmg-notes) from the libdmg-hfsplus project. +# Dependencies + +These are the dependencies used by Bitcoin Core. +You can find installation instructions in the `build-*.md` file for your platform. +"Runtime" and "Version Used" are both in reference to the release binaries. + +| Dependency | Minimum required | +| --- | --- | +| [Autoconf](https://www.gnu.org/software/autoconf/) | [2.69](https://github.com/bitcoin/bitcoin/pull/17769) | +| [Automake](https://www.gnu.org/software/automake/) | [1.13](https://github.com/bitcoin/bitcoin/pull/18290) | +| [Clang](https://clang.llvm.org) | [8.0](https://github.com/bitcoin/bitcoin/pull/24164) | +| [GCC](https://gcc.gnu.org) | [8.1](https://github.com/bitcoin/bitcoin/pull/23060) | +| [Python](https://www.python.org) (tests) | [3.6](https://github.com/bitcoin/bitcoin/pull/19504) | +| [systemtap](https://sourceware.org/systemtap/) ([tracing](tracing.md))| N/A | + +## Required + +| Dependency | Releases | Version used | Minimum required | Runtime | +| --- | --- | --- | --- | --- | +| [Boost](../depends/packages/boost.mk) | [link](https://www.boost.org/users/download/) | [1.77.0](https://github.com/bitcoin/bitcoin/pull/24383) | [1.64.0](https://github.com/bitcoin/bitcoin/pull/22320) | No | +| [libevent](../depends/packages/libevent.mk) | [link](https://github.com/libevent/libevent/releases) | [2.1.12-stable](https://github.com/bitcoin/bitcoin/pull/21991) | [2.1.8](https://github.com/bitcoin/bitcoin/pull/24681) | No | +| glibc | [link](https://www.gnu.org/software/libc/) | N/A | [2.18](https://github.com/bitcoin/bitcoin/pull/23511) | Yes | +| Linux Kernel | [link](https://www.kernel.org/) | N/A | 3.2.0 | Yes | + +## Optional + +### GUI +| Dependency | Releases | Version used | Minimum required | Runtime | +| --- | --- | --- | --- | --- | +| [Fontconfig](../depends/packages/fontconfig.mk) | [link](https://www.freedesktop.org/wiki/Software/fontconfig/) | [2.12.6](https://github.com/bitcoin/bitcoin/pull/23495) | 2.6 | Yes | +| [FreeType](../depends/packages/freetype.mk) | [link](https://freetype.org) | [2.11.0](https://github.com/bitcoin/bitcoin/commit/01544dd78ccc0b0474571da854e27adef97137fb) | 2.3.0 | Yes | +| [qrencode](../depends/packages/qrencode.mk) | [link](https://fukuchi.org/works/qrencode/) | [3.4.4](https://github.com/bitcoin/bitcoin/pull/6373) | | No | +| [Qt](../depends/packages/qt.mk) | [link](https://download.qt.io/official_releases/qt/) | [5.15.3](https://github.com/bitcoin/bitcoin/pull/24668) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | + +### Networking +| Dependency | Releases | Version used | Minimum required | Runtime | +| --- | --- | --- | --- | --- | +| [libnatpmp](../depends/packages/libnatpmp.mk) | [link](https://github.com/miniupnp/libnatpmp/) | commit [4536032...](https://github.com/bitcoin/bitcoin/pull/21209) | | No | +| [MiniUPnPc](../depends/packages/miniupnpc.mk) | [link](https://miniupnp.tuxfamily.org/) | [2.2.2](https://github.com/bitcoin/bitcoin/pull/20421) | 1.9 | No | + +### Notifications +| Dependency | Releases | Version used | Minimum required | Runtime | +| --- | --- | --- | --- | --- | +| [ZeroMQ](../depends/packages/zeromq.mk) | [link](https://github.com/zeromq/libzmq/releases) | [4.3.4](https://github.com/bitcoin/bitcoin/pull/23956) | 4.0.0 | No | + +### Wallet +| Dependency | Releases | Version used | Minimum required | Runtime | +| --- | --- | --- | --- | --- | +| [Berkeley DB](../depends/packages/bdb.mk) (legacy wallet) | [link](https://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html) | 4.8.30 | 4.8.x | No | +| [SQLite](../depends/packages/sqlite.mk) | [link](https://sqlite.org) | [3.32.1](https://github.com/bitcoin/bitcoin/pull/19077) | [3.7.17](https://github.com/bitcoin/bitcoin/pull/19077) | No | diff --git a/doc/descriptors.md b/doc/descriptors.md index 318d065fdb..ab2face4f0 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -33,6 +33,7 @@ Output descriptors currently support: - Pay-to-taproot outputs (P2TR), through the `tr` function. - Multisig scripts, through the `multi` function. - Multisig scripts where the public keys are sorted lexicographically, through the `sortedmulti` function. +- Multisig scripts inside taproot script trees, through the `multi_a` (and `sortedmulti_a`) function. - Any type of supported address through the `addr` function. - Raw hex scripts through the `raw` function. - Public keys (compressed and uncompressed) in hex notation, or BIP32 extended pubkeys with derivation paths. @@ -56,6 +57,7 @@ Output descriptors currently support: - `wsh(multi(1,xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/1/0/*,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/0/0/*))` describes a set of *1-of-2* P2WSH multisig outputs where the first multisig key is the *1/0/`i`* child of the first specified xpub and the second multisig key is the *0/0/`i`* child of the second specified xpub, and `i` is any number in a configurable range (`0-1000` by default). - `wsh(sortedmulti(1,xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/1/0/*,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/0/0/*))` describes a set of *1-of-2* P2WSH multisig outputs where one multisig key is the *1/0/`i`* child of the first specified xpub and the other multisig key is the *0/0/`i`* child of the second specified xpub, and `i` is any number in a configurable range (`0-1000` by default). The order of public keys in the resulting witnessScripts is determined by the lexicographic order of the public keys at that index. - `tr(c6047f9441ed7d6d3045406e95c07cd85c778e4b8cef3ca7abac09b95c709ee5,{pk(fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556),pk(e493dbf1c10d80f3581e4904930b1404cc6c13900ee0758474fa94abe8c4cd13)})` describes a P2TR output with the `c6...` x-only pubkey as internal key, and two script paths. +- `tr(c6047f9441ed7d6d3045406e95c07cd85c778e4b8cef3ca7abac09b95c709ee5,sortedmulti_a(2,2f8bde4d1a07209355b4a7250a5c5128e88b84bddc619ab7cba8d569b240efe4,5cbdf0646e5db4eaa398f365f2ea7a0e3d419b7e0330e39ce92bddedcac4f9bc))` describes a P2TR output with the `c6...` x-only pubkey as internal key, and a single `multi_a` script that needs 2 signatures with 2 specified x-only keys, which will be sorted lexicographically. ## Reference @@ -68,8 +70,10 @@ Descriptors consist of several types of expressions. The top level expression is - `pkh(KEY)` (not inside `tr`): P2PKH output for the given public key (use `addr` if you only know the pubkey hash). - `wpkh(KEY)` (top level or inside `sh` only): P2WPKH output for the given compressed pubkey. - `combo(KEY)` (top level only): an alias for the collection of `pk(KEY)` and `pkh(KEY)`. If the key is compressed, it also includes `wpkh(KEY)` and `sh(wpkh(KEY))`. -- `multi(k,KEY_1,KEY_2,...,KEY_n)` (not inside `tr`): k-of-n multisig script. +- `multi(k,KEY_1,KEY_2,...,KEY_n)` (not inside `tr`): k-of-n multisig script using OP_CHECKMULTISIG. - `sortedmulti(k,KEY_1,KEY_2,...,KEY_n)` (not inside `tr`): k-of-n multisig script with keys sorted lexicographically in the resulting script. +- `multi_a(k,KEY_1,KEY_2,...,KEY_N)` (only inside `tr`): k-of-n multisig script using OP_CHECKSIG, OP_CHECKSIGADD, and OP_NUMEQUAL. +- `sortedmulti_a(k,KEY_1,KEY_2,...,KEY_N)` (only inside `tr`): similar to `multi_a`, but the (x-only) public keys in it will be sorted lexicographically. - `tr(KEY)` or `tr(KEY,TREE)` (top level only): P2TR output with the specified key as internal key, and optionally a tree of script paths. - `addr(ADDR)` (top level only): the script which ADDR expands to. - `raw(HEX)` (top level only): the script whose hex encoding is HEX. diff --git a/doc/developer-notes.md b/doc/developer-notes.md index bfb64093e1..9b1026a375 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -17,6 +17,7 @@ Developer Notes - [`debug.log`](#debuglog) - [Signet, testnet, and regtest modes](#signet-testnet-and-regtest-modes) - [DEBUG_LOCKORDER](#debug_lockorder) + - [DEBUG_LOCKCONTENTION](#debug_lockcontention) - [Valgrind suppressions file](#valgrind-suppressions-file) - [Compiling for test coverage](#compiling-for-test-coverage) - [Performance profiling with perf](#performance-profiling-with-perf) @@ -137,11 +138,57 @@ public: } // namespace foo ``` +Coding Style (C++ named arguments) +------------------------------ + +When passing named arguments, use a format that clang-tidy understands. The +argument names can otherwise not be verified by clang-tidy. + +For example: + +```c++ +void function(Addrman& addrman, bool clear); + +int main() +{ + function(g_addrman, /*clear=*/false); +} +``` + +### Running clang-tidy + +To run clang-tidy on Ubuntu/Debian, install the dependencies: + +```sh +apt install clang-tidy bear clang +``` + +Then, pass clang as compiler to configure, and use bear to produce the `compile_commands.json`: + +```sh +./autogen.sh && ./configure CC=clang CXX=clang++ +make clean && bear make -j $(nproc) # For bear 2.x +make clean && bear -- make -j $(nproc) # For bear 3.x +``` + +To run clang-tidy on all source files: + +```sh +( cd ./src/ && run-clang-tidy -j $(nproc) ) +``` + +To run clang-tidy on the changed source lines: + +```sh +git diff | ( cd ./src/ && clang-tidy-diff -p2 -j $(nproc) ) +``` + Coding Style (Python) --------------------- Refer to [/test/functional/README.md#style-guidelines](/test/functional/README.md#style-guidelines). + Coding Style (Doxygen-compatible comments) ------------------------------------------ @@ -316,6 +363,19 @@ configure option adds `-DDEBUG_LOCKORDER` to the compiler flags. This inserts run-time checks to keep track of which locks are held and adds warnings to the `debug.log` file if inconsistencies are detected. +### DEBUG_LOCKCONTENTION + +Defining `DEBUG_LOCKCONTENTION` adds a "lock" logging category to the logging +RPC that, when enabled, logs the location and duration of each lock contention +to the `debug.log` file. + +To enable it, run configure with `-DDEBUG_LOCKCONTENTION` added to your +CPPFLAGS, e.g. `CPPFLAGS="-DDEBUG_LOCKCONTENTION"`, then build and run bitcoind. + +You can then use the `-debug=lock` configuration option at bitcoind startup or +`bitcoin-cli logging '["lock"]'` at runtime to turn on lock contention logging. +It can be toggled off again with `bitcoin-cli logging [] '["lock"]'`. + ### Assertions and Checks The util file `src/util/check.h` offers helpers to protect against coding and @@ -1160,7 +1220,10 @@ A few guidelines for introducing and reviewing new RPC interfaces: - *Rationale*: Consistency with the existing interface. -- Argument naming: use snake case `fee_delta` (and not, e.g. camel case `feeDelta`) +- Argument and field naming: please consider whether there is already a naming + style or spelling convention in the API for the type of object in question + (`blockhash`, for example), and if so, try to use that. If not, use snake case + `fee_delta` (and not, e.g. `feedelta` or camel case `feeDelta`). - *Rationale*: Consistency with the existing interface. diff --git a/doc/i2p.md b/doc/i2p.md index ee650f3999..39f65c4e5f 100644 --- a/doc/i2p.md +++ b/doc/i2p.md @@ -65,9 +65,9 @@ logging` for more information. -onlynet=i2p ``` -Make outgoing connections only to I2P addresses. Incoming connections are not -affected by this option. It can be specified multiple times to allow multiple -network types, e.g. onlynet=onion, onlynet=i2p. +Make automatic outbound connections only to I2P addresses. Inbound and manual +connections are not affected by this option. It can be specified multiple times +to allow multiple networks, e.g. onlynet=onion, onlynet=i2p. I2P support was added to Bitcoin Core in version 22.0 and there may be fewer I2P peers than Tor or IP ones. Therefore, using I2P alone without other networks may @@ -80,15 +80,15 @@ phase when syncing up a new node can be very slow. This phase can be sped up by using other networks, for instance `onlynet=onion`, at the same time. In general, a node can be run with both onion and I2P hidden services (or -any/all of IPv4/IPv6/onion/I2P), which can provide a potential fallback if one -of the networks has issues. +any/all of IPv4/IPv6/onion/I2P/CJDNS), which can provide a potential fallback if +one of the networks has issues. ## I2P-related information in Bitcoin Core There are several ways to see your I2P address in Bitcoin Core: -- in the debug log (grep for `AddLocal`, the I2P address ends in `.b32.i2p`) -- in the output of the `getnetworkinfo` RPC in the "localaddresses" section -- in the output of `bitcoin-cli -netinfo` peer connections dashboard +- in the "Local addresses" output of CLI `-netinfo` +- in the "localaddresses" output of RPC `getnetworkinfo` +- in the debug log (grep for `AddLocal`; the I2P address ends in `.b32.i2p`) To see which I2P peers your node is connected to, use `bitcoin-cli -netinfo 4` or the `getpeerinfo` RPC (e.g. `bitcoin-cli getpeerinfo`). diff --git a/doc/multisig-tutorial.md b/doc/multisig-tutorial.md index 0793040418..1d2b3244bc 100644 --- a/doc/multisig-tutorial.md +++ b/doc/multisig-tutorial.md @@ -29,7 +29,7 @@ These three wallets should not be used directly for privacy reasons (public key ```bash for ((n=1;n<=3;n++)) do - ./src/bitcoin-cli -signet -named createwallet wallet_name="participant_${n}" descriptors=true + ./src/bitcoin-cli -signet createwallet "participant_${n}" done ``` @@ -109,7 +109,7 @@ Then import the descriptors created in the previous step using the `importdescri After that, `getwalletinfo` can be used to check if the wallet was created successfully. ```bash -./src/bitcoin-cli -signet -named createwallet wallet_name="multisig_wallet_01" disable_private_keys=true blank=true descriptors=true +./src/bitcoin-cli -signet -named createwallet wallet_name="multisig_wallet_01" disable_private_keys=true blank=true ./src/bitcoin-cli -signet -rpcwallet="multisig_wallet_01" importdescriptors "$multisig_desc" @@ -238,4 +238,4 @@ psbt_2=$(./src/bitcoin-cli -signet -rpcwallet="participant_2" walletprocesspsbt finalized_psbt_hex=$(./src/bitcoin-cli -signet finalizepsbt $psbt_2 | jq -r '.hex') ./src/bitcoin-cli -signet sendrawtransaction $finalized_psbt_hex -```
\ No newline at end of file +``` diff --git a/doc/p2p-bad-ports.md b/doc/p2p-bad-ports.md index 0dd7d36cf4..4f717f97a2 100644 --- a/doc/p2p-bad-ports.md +++ b/doc/p2p-bad-ports.md @@ -1,6 +1,6 @@ -When Bitcoin Core automatically opens outgoing P2P connections it chooses +When Bitcoin Core automatically opens outgoing P2P connections, it chooses a peer (address and port) from its list of potential peers. This list is -populated with unchecked data, gossiped over the P2P network by other peers. +populated with unchecked data gossiped over the P2P network by other peers. A malicious actor may gossip an address:port where no Bitcoin node is listening, or one where a service is listening that is not related to the Bitcoin network. @@ -17,7 +17,7 @@ authentication are unlikely to be considered a malicious action, e.g. port 80 (http). Below is a list of "bad" ports which Bitcoin Core avoids when choosing a peer to -connect to. If a node is listening on such a port, it will likely receive less +connect to. If a node is listening on such a port, it will likely receive fewer incoming connections. 1: tcpmux diff --git a/doc/policy/packages.md b/doc/policy/packages.md index 7f7fbe18cd..f2a3d6517e 100644 --- a/doc/policy/packages.md +++ b/doc/policy/packages.md @@ -72,3 +72,48 @@ test accepts): a competing package or transaction with a mutated witness, even though the two same-txid-different-witness transactions are conflicting and cannot replace each other, the honest package should still be considered for acceptance. + +### Package Fees and Feerate + +*Package Feerate* is the total modified fees (base fees + any fee delta from +`prioritisetransaction`) divided by the total virtual size of all transactions in the package. +If any transactions in the package are already in the mempool, they are not submitted again +("deduplicated") and are thus excluded from this calculation. + +To meet the two feerate requirements of a mempool, i.e., the pre-configured minimum relay feerate +(`minRelayTxFee`) and the dynamic mempool minimum feerate, the total package feerate is used instead +of the individual feerate. The individual transactions are allowed to be below the feerate +requirements if the package meets the feerate requirements. For example, the parent(s) in the +package can pay no fees but be paid for by the child. + +*Rationale*: This can be thought of as "CPFP within a package," solving the issue of a parent not +meeting minimum fees on its own. This would allow contracting applications to adjust their fees at +broadcast time instead of overshooting or risking becoming stuck or pinned. + +*Rationale*: It would be incorrect to use the fees of transactions that are already in the mempool, as +we do not want a transaction's fees to be double-counted. + +Implementation Note: Transactions within a package are always validated individually first, and +package validation is used for the transactions that failed. Since package feerate is only +calculated using transactions that are not in the mempool, this implementation detail affects the +outcome of package validation. + +*Rationale*: Packages are intended for incentive-compatible fee-bumping: transaction B is a +"legitimate" fee-bump for transaction A only if B is a descendant of A and has a *higher* feerate +than A. We want to prevent "parents pay for children" behavior; fees of parents should not help +their children, since the parents can be mined without the child. More generally, if transaction A +is not needed in order for transaction B to be mined, A's fees cannot help B. In a +child-with-parents package, simply excluding any parent transactions that meet feerate requirements +individually is sufficient to ensure this. + +*Rationale*: We must not allow a low-feerate child to prevent its parent from being accepted; fees +of children should not negatively impact their parents, since they are not necessary for the parents +to be mined. More generally, if transaction B is not needed in order for transaction A to be mined, +B's fees cannot harm A. In a child-with-parents package, simply validating parents individually +first is sufficient to ensure this. + +*Rationale*: As a principle, we want to avoid accidentally restricting policy in order to be +backward-compatible for users and applications that rely on p2p transaction relay. Concretely, +package validation should not prevent the acceptance of a transaction that would otherwise be +policy-valid on its own. By always accepting a transaction that passes individual validation before +trying package validation, we prevent any unintentional restriction of policy. diff --git a/doc/release-notes-24098.md b/doc/release-notes-24098.md new file mode 100644 index 0000000000..79e047e9a5 --- /dev/null +++ b/doc/release-notes-24098.md @@ -0,0 +1,22 @@ +Notable changes +=============== + +Updated REST APIs +----------------- + +- The `/headers/` and `/blockfilterheaders/` endpoints have been updated to use + a query parameter instead of path parameter to specify the result count. The + count parameter is now optional, and defaults to 5 for both endpoints. The old + endpoints are still functional, and have no documented behaviour change. + + For `/headers`, use + `GET /rest/headers/<BLOCK-HASH>.<bin|hex|json>?count=<COUNT=5>` + instead of + `GET /rest/headers/<COUNT>/<BLOCK-HASH>.<bin|hex|json>` (deprecated) + + For `/blockfilterheaders/`, use + `GET /rest/blockfilterheaders/<FILTERTYPE>/<BLOCK-HASH>.<bin|hex|json>?count=<COUNT=5>` + instead of + `GET /rest/blockfilterheaders/<FILTERTYPE>/<COUNT>/<BLOCK-HASH>.<bin|hex|json>` (deprecated) + + (#24098) diff --git a/doc/release-notes-24118.md b/doc/release-notes-24118.md new file mode 100644 index 0000000000..16f23c7d00 --- /dev/null +++ b/doc/release-notes-24118.md @@ -0,0 +1,10 @@ +New RPCs +-------- + +- The `sendall` RPC spends specific UTXOs to one or more recipients + without creating change. By default, the `sendall` RPC will spend + every UTXO in the wallet. `sendall` is useful to empty wallets or to + create a changeless payment from select UTXOs. When creating a payment + from a specific amount for which the recipient incurs the transaction + fee, continue to use the `subtractfeefromamount` option via the + `send`, `sendtoaddress`, or `sendmany` RPCs. (#24118) diff --git a/doc/release-notes-24198.md b/doc/release-notes-24198.md new file mode 100644 index 0000000000..e41b2a8e26 --- /dev/null +++ b/doc/release-notes-24198.md @@ -0,0 +1,6 @@ +Updated RPCs +------------ + +- The `listtransactions`, `gettransaction`, and `listsinceblock` + RPC methods now include a wtxid field (hash of serialized transaction, + including witness data) for each transaction.
\ No newline at end of file diff --git a/doc/release-notes-24494.md b/doc/release-notes-24494.md new file mode 100644 index 0000000000..afbb926433 --- /dev/null +++ b/doc/release-notes-24494.md @@ -0,0 +1,2 @@ +To help prevent fingerprinting transactions created by the Bitcoin Core wallet, change output +amounts are now randomized. (#24494) diff --git a/doc/release-notes-empty-template.md b/doc/release-notes-empty-template.md new file mode 100644 index 0000000000..8462714898 --- /dev/null +++ b/doc/release-notes-empty-template.md @@ -0,0 +1,99 @@ +*The release notes draft is a temporary file that can be added to by anyone. See +[/doc/developer-notes.md#release-notes](/doc/developer-notes.md#release-notes) +for the process.* + +*version* Release Notes Draft +=============================== + +Bitcoin Core version *version* is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-*version*/> + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +P2P and network changes +----------------------- + +Updated RPCs +------------ + + +Changes to wallet related RPCs can be found in the Wallet section below. + +New RPCs +-------- + +Build System +------------ + +Updated settings +---------------- + + +Changes to GUI or wallet related settings can be found in the GUI or Wallet section below. + +New settings +------------ + +Tools and Utilities +------------------- + +Wallet +------ + +GUI changes +----------- + +Low-level changes +================= + +RPC +--- + +Tests +----- + +*version* change log +==================== + +Credits +======= + +Thanks to everyone who directly contributed to this release: + + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes.md b/doc/release-notes.md index 5d1aa85593..8462714898 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -1,17 +1,7 @@ -*After branching off for a major version release of Bitcoin Core, use this -template to create the initial release notes draft.* - *The release notes draft is a temporary file that can be added to by anyone. See [/doc/developer-notes.md#release-notes](/doc/developer-notes.md#release-notes) for the process.* -*Create the draft, named* "*version* Release Notes Draft" -*(e.g. "22.0 Release Notes Draft"), as a collaborative wiki in:* - -https://github.com/bitcoin-core/bitcoin-devwiki/wiki/ - -*Before the final release, move the notes back to this git repository.* - *version* Release Notes Draft =============================== @@ -57,170 +47,47 @@ Notable changes P2P and network changes ----------------------- -- A bitcoind node will no longer rumour addresses to inbound peers by default. - They will become eligible for address gossip after sending an ADDR, ADDRV2, - or GETADDR message. (#21528) - -Fee estimation changes ----------------------- - -- Fee estimation now takes the feerate of replacement (RBF) transactions into - account. (#22539) - -Rescan startup parameter removed --------------------------------- - -The `-rescan` startup parameter has been removed. Wallets which require -rescanning due to corruption will still be rescanned on startup. -Otherwise, please use the `rescanblockchain` RPC to trigger a rescan. (#23123) - Updated RPCs ------------ -- The `validateaddress` RPC now returns an `error_locations` array for invalid - addresses, with the indices of invalid character locations in the address (if - known). For example, this will attempt to locate up to two Bech32 errors, and - return their locations if successful. Success and correctness are only guaranteed - if fewer than two substitution errors have been made. - The error message returned in the `error` field now also returns more specific - errors when decoding fails. (#16807) - -- The `-deprecatedrpc=addresses` configuration option has been removed. RPCs - `gettxout`, `getrawtransaction`, `decoderawtransaction`, `decodescript`, - `gettransaction verbose=true` and REST endpoints `/rest/tx`, `/rest/getutxos`, - `/rest/block` no longer return the `addresses` and `reqSigs` fields, which - were previously deprecated in 22.0. (#22650) -- The `getblock` RPC command now supports verbosity level 3 containing transaction inputs' - `prevout` information. The existing `/rest/block/` REST endpoint is modified to contain - this information too. Every `vin` field will contain an additional `prevout` subfield - describing the spent output. `prevout` contains the following keys: - - `generated` - true if the spent coins was a coinbase. - - `height` - - `value` - - `scriptPubKey` - -- The top-level fee fields `fee`, `modifiedfee`, `ancestorfees` and `descendantfees` - returned by RPCs `getmempoolentry`,`getrawmempool(verbose=true)`, - `getmempoolancestors(verbose=true)` and `getmempooldescendants(verbose=true)` - are deprecated and will be removed in the next major version (use - `-deprecated=fees` if needed in this version). The same fee fields can be accessed - through the `fees` object in the result. WARNING: deprecated - fields `ancestorfees` and `descendantfees` are denominated in sats, whereas all - fields in the `fees` object are denominated in BTC. (#22689) - -- Both `createmultisig` and `addmultisigaddress` now include a `warnings` - field, which will show a warning if a non-legacy address type is requested - when using uncompressed public keys. (#23113) + +Changes to wallet related RPCs can be found in the Wallet section below. New RPCs -------- -- Information on soft fork status has been moved from `getblockchaininfo` - to the new `getdeploymentinfo` RPC which allows querying soft fork status at any - block, rather than just at the chain tip. Inclusion of soft fork - status in `getblockchaininfo` can currently be restored using the - configuration `-deprecatedrpc=softforks`, but this will be removed in - a future release. Note that in either case, the `status` field - now reflects the status of the current block rather than the next - block. (#23508) - Build System ------------ -Files ------ - -* On startup, the list of banned hosts and networks (via `setban` RPC) in - `banlist.dat` is ignored and only `banlist.json` is considered. Bitcoin Core - version 22.x is the only version that can read `banlist.dat` and also write - it to `banlist.json`. If `banlist.json` already exists, version 22.x will not - try to translate the `banlist.dat` into json. After an upgrade, `listbanned` - can be used to double check the parsed entries. (#22570) - -New settings ------------- - Updated settings ---------------- -- In previous releases, the meaning of the command line option - `-persistmempool` (without a value provided) incorrectly disabled mempool - persistence. `-persistmempool` is now treated like other boolean options to - mean `-persistmempool=1`. Passing `-persistmempool=0`, `-persistmempool=1` - and `-nopersistmempool` is unaffected. (#23061) -- `-maxuploadtarget` now allows human readable byte units [k|K|m|M|g|G|t|T]. - E.g. `-maxuploadtarget=500g`. No whitespace, +- or fractions allowed. - Default is `M` if no suffix provided. (#23249) +Changes to GUI or wallet related settings can be found in the GUI or Wallet section below. -- If `-proxy=` is given together with `-noonion` then the provided proxy will - not be set as a proxy for reaching the Tor network. So it will not be - possible to open manual connections to the Tor network for example with the - `addnode` RPC. To mimic the old behavior use `-proxy=` together with - `-onlynet=` listing all relevant networks except `onion`. (#22834) +New settings +------------ Tools and Utilities ------------------- -- Update `-getinfo` to return data in a user-friendly format that also reduces vertical space. (#21832) - -- CLI `-addrinfo` now returns a single field for the number of `onion` addresses - known to the node instead of separate `torv2` and `torv3` fields, as support - for Tor V2 addresses was removed from Bitcoin Core in 22.0. (#22544) - Wallet ------ -- `upgradewallet` will now automatically flush the keypool if upgrading - from a non-HD wallet to an HD wallet, to immediately start using the - newly-generated HD keys. (#23093) - -- a new RPC `newkeypool` has been added, which will flush (entirely - clear and refill) the keypool. (#23093) - -- `listunspent` now includes `ancestorcount`, `ancestorsize`, and - `ancestorfees` for each transaction output that is still in the mempool. - (#12677) - -- `lockunspent` now optionally takes a third parameter, `persistent`, which - causes the lock to be written persistently to the wallet database. This - allows UTXOs to remain locked even after node restarts or crashes. (#23065) - -- `receivedby` RPCs now include coinbase transactions. Previously, the - following wallet RPCs excluded coinbase transactions: `getreceivedbyaddress`, - `getreceivedbylabel`, `listreceivedbyaddress`, `listreceivedbylabel`. This - release changes this behaviour and returns results accounting for received - coins from coinbase outputs. The previous behaviour can be restored using the - configuration `-deprecatedrpc=exclude_coinbase`, but may be removed in a - future release. (#14707) - -- A new option in the same `receivedby` RPCs, `include_immature_coinbase` - (default=`false`), determines whether to account for immature coinbase - transactions. Immature coinbase transactions are coinbase transactions that - have 100 or fewer confirmations, and are not spendable. (#14707) - GUI changes ----------- -- UTXOs which are locked via the GUI are now stored persistently in the - wallet database, so are not lost on node shutdown or crash. (#23065) - -- The Bech32 checkbox has been replaced with a dropdown for all address types, including the new Bech32m (BIP-350) standard for Taproot enabled wallets. - Low-level changes ================= RPC --- -- `getblockchaininfo` now returns a new `time` field, that provides the chain tip time. (#22407) - Tests ----- -- For the `regtest` network the activation heights of several softforks were - set to block height 1. They can be changed by the runtime setting - `-testactivationheight=name@height`. (#22818) +*version* change log +==================== Credits ======= diff --git a/doc/release-notes/release-notes-23.0.md b/doc/release-notes/release-notes-23.0.md new file mode 100644 index 0000000000..b1467a0f71 --- /dev/null +++ b/doc/release-notes/release-notes-23.0.md @@ -0,0 +1,373 @@ +23.0 Release Notes +================== + +Bitcoin Core version 23.0 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-23.0/> + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +P2P and network changes +----------------------- + +- A bitcoind node will no longer rumour addresses to inbound peers by default. + They will become eligible for address gossip after sending an ADDR, ADDRV2, + or GETADDR message. (#21528) + +- Before this release, Bitcoin Core had a strong preference to try to connect only to peers that listen on port 8333. As a result of that, Bitcoin nodes listening on non-standard ports would likely not get any Bitcoin Core peers connecting to them. This preference has been removed. (#23542) + +- Full support has been added for the CJDNS network. See the new option `-cjdnsreachable` and [doc/cjdns.md](https://github.com/bitcoin/bitcoin/tree/23.x/doc/cjdns.md) (#23077) + +Fee estimation changes +---------------------- + +- Fee estimation now takes the feerate of replacement (RBF) transactions into + account. (#22539) + +Rescan startup parameter removed +-------------------------------- + +The `-rescan` startup parameter has been removed. Wallets which require +rescanning due to corruption will still be rescanned on startup. +Otherwise, please use the `rescanblockchain` RPC to trigger a rescan. (#23123) + +Tracepoints and Userspace, Statically Defined Tracing support +------------------------------------------------------------- + +Bitcoin Core release binaries for Linux now include experimental tracepoints which +act as an interface for process-internal events. These can be used for review, +debugging, monitoring, and more. The tracepoint API is semi-stable. While the API +is tested, process internals might change between releases requiring changes to the +tracepoints. Information about the existing tracepoints can be found under +[doc/tracing.md](https://github.com/bitcoin/bitcoin/blob/23.x/doc/tracing.md) and +usage examples are provided in [contrib/tracing/](https://github.com/bitcoin/bitcoin/tree/23.x/contrib/tracing). + +Updated RPCs +------------ + +- The `validateaddress` RPC now returns an `error_locations` array for invalid + addresses, with the indices of invalid character locations in the address (if + known). For example, this will attempt to locate up to two Bech32 errors, and + return their locations if successful. Success and correctness are only guaranteed + if fewer than two substitution errors have been made. + The error message returned in the `error` field now also returns more specific + errors when decoding fails. (#16807) + +- The `-deprecatedrpc=addresses` configuration option has been removed. RPCs + `gettxout`, `getrawtransaction`, `decoderawtransaction`, `decodescript`, + `gettransaction verbose=true` and REST endpoints `/rest/tx`, `/rest/getutxos`, + `/rest/block` no longer return the `addresses` and `reqSigs` fields, which + were previously deprecated in 22.0. (#22650) +- The `getblock` RPC command now supports verbosity level 3 containing transaction inputs' + `prevout` information. The existing `/rest/block/` REST endpoint is modified to contain + this information too. Every `vin` field will contain an additional `prevout` subfield + describing the spent output. `prevout` contains the following keys: + - `generated` - true if the spent coins was a coinbase. + - `height` + - `value` + - `scriptPubKey` + +- The top-level fee fields `fee`, `modifiedfee`, `ancestorfees` and `descendantfees` + returned by RPCs `getmempoolentry`,`getrawmempool(verbose=true)`, + `getmempoolancestors(verbose=true)` and `getmempooldescendants(verbose=true)` + are deprecated and will be removed in the next major version (use + `-deprecated=fees` if needed in this version). The same fee fields can be accessed + through the `fees` object in the result. WARNING: deprecated + fields `ancestorfees` and `descendantfees` are denominated in sats, whereas all + fields in the `fees` object are denominated in BTC. (#22689) + +- Both `createmultisig` and `addmultisigaddress` now include a `warnings` + field, which will show a warning if a non-legacy address type is requested + when using uncompressed public keys. (#23113) + +Changes to wallet related RPCs can be found in the Wallet section below. + +New RPCs +-------- + +- Information on soft fork status has been moved from `getblockchaininfo` + to the new `getdeploymentinfo` RPC which allows querying soft fork status at any + block, rather than just at the chain tip. Inclusion of soft fork + status in `getblockchaininfo` can currently be restored using the + configuration `-deprecatedrpc=softforks`, but this will be removed in + a future release. Note that in either case, the `status` field + now reflects the status of the current block rather than the next + block. (#23508) + +Files +----- + +* On startup, the list of banned hosts and networks (via `setban` RPC) in + `banlist.dat` is ignored and only `banlist.json` is considered. Bitcoin Core + version 22.x is the only version that can read `banlist.dat` and also write + it to `banlist.json`. If `banlist.json` already exists, version 22.x will not + try to translate the `banlist.dat` into json. After an upgrade, `listbanned` + can be used to double check the parsed entries. (#22570) + +Updated settings +---------------- + +- In previous releases, the meaning of the command line option + `-persistmempool` (without a value provided) incorrectly disabled mempool + persistence. `-persistmempool` is now treated like other boolean options to + mean `-persistmempool=1`. Passing `-persistmempool=0`, `-persistmempool=1` + and `-nopersistmempool` is unaffected. (#23061) + +- `-maxuploadtarget` now allows human readable byte units [k|K|m|M|g|G|t|T]. + E.g. `-maxuploadtarget=500g`. No whitespace, +- or fractions allowed. + Default is `M` if no suffix provided. (#23249) + +- If `-proxy=` is given together with `-noonion` then the provided proxy will + not be set as a proxy for reaching the Tor network. So it will not be + possible to open manual connections to the Tor network for example with the + `addnode` RPC. To mimic the old behavior use `-proxy=` together with + `-onlynet=` listing all relevant networks except `onion`. (#22834) + +Tools and Utilities +------------------- + +- Update `-getinfo` to return data in a user-friendly format that also reduces vertical space. (#21832) + +- CLI `-addrinfo` now returns a single field for the number of `onion` addresses + known to the node instead of separate `torv2` and `torv3` fields, as support + for Tor V2 addresses was removed from Bitcoin Core in 22.0. (#22544) + +Wallet +------ + +- Descriptor wallets are now the default wallet type. Newly created wallets + will use descriptors unless `descriptors=false` is set during `createwallet`, or + the `Descriptor wallet` checkbox is unchecked in the GUI. + + Note that wallet RPC commands like `importmulti` and `dumpprivkey` cannot be + used with descriptor wallets, so if your client code relies on these commands + without specifying `descriptors=false` during wallet creation, you will need + to update your code. + +- Newly created descriptor wallets will contain an automatically generated `tr()` + descriptor which allows for creating single key Taproot receiving addresses. + +- `upgradewallet` will now automatically flush the keypool if upgrading + from a non-HD wallet to an HD wallet, to immediately start using the + newly-generated HD keys. (#23093) + +- a new RPC `newkeypool` has been added, which will flush (entirely + clear and refill) the keypool. (#23093) + +- `listunspent` now includes `ancestorcount`, `ancestorsize`, and + `ancestorfees` for each transaction output that is still in the mempool. + (#12677) + +- `lockunspent` now optionally takes a third parameter, `persistent`, which + causes the lock to be written persistently to the wallet database. This + allows UTXOs to remain locked even after node restarts or crashes. (#23065) + +- `receivedby` RPCs now include coinbase transactions. Previously, the + following wallet RPCs excluded coinbase transactions: `getreceivedbyaddress`, + `getreceivedbylabel`, `listreceivedbyaddress`, `listreceivedbylabel`. This + release changes this behaviour and returns results accounting for received + coins from coinbase outputs. The previous behaviour can be restored using the + configuration `-deprecatedrpc=exclude_coinbase`, but may be removed in a + future release. (#14707) + +- A new option in the same `receivedby` RPCs, `include_immature_coinbase` + (default=`false`), determines whether to account for immature coinbase + transactions. Immature coinbase transactions are coinbase transactions that + have 100 or fewer confirmations, and are not spendable. (#14707) + +GUI changes +----------- + +- UTXOs which are locked via the GUI are now stored persistently in the + wallet database, so are not lost on node shutdown or crash. (#23065) + +- The Bech32 checkbox has been replaced with a dropdown for all address types, including the new Bech32m (BIP-350) standard for Taproot enabled wallets. + +Low-level changes +================= + +RPC +--- + +- `getblockchaininfo` now returns a new `time` field, that provides the chain tip time. (#22407) + +Tests +----- + +- For the `regtest` network the activation heights of several softforks were + set to block height 1. They can be changed by the runtime setting + `-testactivationheight=name@height`. (#22818) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 0xb10c +- 0xree +- Aaron Clauson +- Adrian-Stefan Mares +- agroce +- aitorjs +- Alex Groce +- amadeuszpawlik +- Amiti Uttarwar +- Andrew Chow +- Andrew Poelstra +- Andrew Toth +- anouar kappitou +- Anthony Towns +- Antoine Poinsot +- Arnab Sen +- Ben Woosley +- benthecarman +- Bitcoin Hodler +- BitcoinTsunami +- brianddk +- Bruno Garcia +- CallMeMisterOwl +- Calvin Kim +- Carl Dong +- Cory Fields +- Cuong V. Nguyen +- Darius Parvin +- Dhruv Mehta +- Dimitri Deijs +- Dimitris Apostolou +- Dmitry Goncharov +- Douglas Chimento +- eugene +- Fabian Jahr +- fanquake +- Florian Baumgartl +- fyquah +- Gleb Naumenko +- glozow +- Gregory Sanders +- Heebs +- Hennadii Stepanov +- hg333 +- HiLivin +- Igor Cota +- Jadi +- James O'Beirne +- Jameson Lopp +- Jarol Rodriguez +- Jeremy Rand +- Jeremy Rubin +- Joan Karadimov +- John Newbery +- Jon Atack +- João Barbosa +- josibake +- junderw +- Karl-Johan Alm +- katesalazar +- Kennan Mell +- Kiminuo +- Kittywhiskers Van Gogh +- Klement Tan +- Kristaps Kaupe +- Kuro +- Larry Ruane +- lsilva01 +- lucash-dev +- Luke Dashjr +- MarcoFalke +- Martin Leitner-Ankerl +- Martin Zumsande +- Matt Corallo +- Matt Whitlock +- MeshCollider +- Michael Dietz +- Murch +- naiza +- Nathan Garabedian +- Nelson Galdeman +- NikhilBartwal +- Niklas Gögge +- node01 +- nthumann +- Pasta +- Patrick Kamin +- Pavel Safronov +- Pavol Rusnak +- Perlover +- Pieter Wuille +- practicalswift +- pradumnasaraf +- pranabp-bit +- Prateek Sancheti +- Prayank +- Rafael Sadowski +- rajarshimaitra +- randymcmillan +- ritickgoenka +- Rob Fielding +- Rojar Smith +- Russell Yanofsky +- S3RK +- Saibato +- Samuel Dobson +- sanket1729 +- seaona +- Sebastian Falbesoner +- sh15h4nk +- Shashwat +- Shorya +- ShubhamPalriwala +- Shubhankar Gambhir +- Sjors Provoost +- sogoagain +- sstone +- stratospher +- Suriyaa Rocky Sundararuban +- Taeik Lim +- TheCharlatan +- Tim Ruffing +- Tobin Harding +- Troy Giorshev +- Tyler Chambers +- Vasil Dimov +- W. J. van der Laan +- w0xlt +- willcl-ark +- William Casarin +- zealsham +- Zero-1729 + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-process.md b/doc/release-process.md index 5a74f72b6e..03cea6f194 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -8,6 +8,7 @@ Release Process * Update translations see [translation_process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md#synchronising-translations). * Update release candidate version in `configure.ac` (`CLIENT_VERSION_RC`). * Update manpages (after rebuilding the binaries), see [gen-manpages.py](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-manpagespy). +* Update bitcoin.conf and commit, see [gen-bitcoin-conf.sh](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-bitcoin-confsh). ### Before every major and minor release @@ -37,7 +38,8 @@ Release Process - This update should be reviewed with a reindex-chainstate with assumevalid=0 to catch any defect that causes rejection of blocks in the past history. - Clear the release notes and move them to the wiki (see "Write the release notes" below). -- Translations on Transifex +- Translations on Transifex: + - Pull translations from Transifex into the master branch. - Create [a new resource](https://www.transifex.com/bitcoin/bitcoin/content/) named after the major version with the slug `[bitcoin.qt-translation-<RRR>x]`, where `RRR` is the major branch number padded with zeros. Use `src/qt/locale/bitcoin_en.xlf` to create it. - In the project workflow settings, ensure that [Translation Memory Fill-up](https://docs.transifex.com/translation-memory/enabling-autofill) is enabled and that [Translation Memory Context Matching](https://docs.transifex.com/translation-memory/translation-memory-with-context) is disabled. - Update the Transifex slug in [`.tx/config`](/.tx/config) to the slug of the resource created in the first step. This identifies which resource the translations will be synchronized from. @@ -47,13 +49,15 @@ Release Process #### After branch-off (on the major release branch) - Update the versions. +- Create the draft, named "*version* Release Notes Draft", as a [collaborative wiki](https://github.com/bitcoin-core/bitcoin-devwiki/wiki/_new). +- Clear the release notes: `cp doc/release-notes-empty-template.md doc/release-notes.md` - Create a pinned meta-issue for testing the release candidate (see [this issue](https://github.com/bitcoin/bitcoin/issues/17079) for an example) and provide a link to it in the release announcements where useful. - Translations on Transifex - Change the auto-update URL for the new major version's resource away from `master` and to the branch, e.g. `https://raw.githubusercontent.com/bitcoin/bitcoin/<branch>/src/qt/locale/bitcoin_en.xlf`. Do not forget this or it will keep tracking the translations on master instead, drifting away from the specific major release. #### Before final release -- Merge the release notes from the wiki into the branch. +- Merge the release notes from [the wiki](https://github.com/bitcoin-core/bitcoin-devwiki/wiki/) into the branch. - Ensure the "Needs release note" label is removed from all relevant pull requests and issues. #### Tagging a release (candidate) @@ -110,28 +114,24 @@ against other `guix-attest` signatures. git -C ./guix.sigs pull ``` -### Create the macOS SDK tarball: (first time, or when SDK version changes) +### Create the macOS SDK tarball (first time, or when SDK version changes) Create the macOS SDK tarball, see the [macdeploy instructions](/contrib/macdeploy/README.md#deterministic-macos-dmg-notes) for details. -### Build and attest to build outputs: +### Build and attest to build outputs Follow the relevant Guix README.md sections: - [Building](/contrib/guix/README.md#building) - [Attesting to build outputs](/contrib/guix/README.md#attesting-to-build-outputs) -### Verify other builders' signatures to your own. (Optional) - -Add other builders keys to your gpg keyring, and/or refresh keys: See `../bitcoin/contrib/builder-keys/README.md`. +### Verify other builders' signatures to your own (optional) -Follow the relevant Guix README.md sections: +- [Add other builders keys to your gpg keyring, and/or refresh keys](/contrib/builder-keys/README.md) - [Verifying build output attestations](/contrib/guix/README.md#verifying-build-output-attestations) -### Next steps: - -Commit your signature to guix.sigs: +### Commit your non codesigned signature to guix.sigs ```sh pushd ./guix.sigs @@ -141,29 +141,27 @@ git push # Assuming you can push to the guix.sigs tree popd ``` -Codesigner only: Create Windows/macOS detached signatures: -- Only one person handles codesigning. Everyone else should skip to the next step. -- Only once the Windows/macOS builds each have 3 matching signatures may they be signed with their respective release keys. +## Codesigning -Codesigner only: Sign the macOS binary: +### macOS codesigner only: Create detached macOS signatures (assuming [signapple](https://github.com/achow101/signapple/) is installed and up to date with master branch) - transfer bitcoin-osx-unsigned.tar.gz to macOS for signing tar xf bitcoin-osx-unsigned.tar.gz - ./detached-sig-create.sh -s "Key ID" + ./detached-sig-create.sh /path/to/codesign.p12 Enter the keychain password and authorize the signature - Move signature-osx.tar.gz back to the guix-build host + signature-osx.tar.gz will be created -Codesigner only: Sign the windows binaries: +### Windows codesigner only: Create detached Windows signatures tar xf bitcoin-win-unsigned.tar.gz ./detached-sig-create.sh -key /path/to/codesign.key Enter the passphrase for the key when prompted signature-win.tar.gz will be created -Code-signer only: It is advised to test that the code signature attaches properly prior to tagging by performing the `guix-codesign` step. +### Windows and macOS codesigners only: test code signatures +It is advised to test that the code signature attaches properly prior to tagging by performing the `guix-codesign` step. However if this is done, once the release has been tagged in the bitcoin-detached-sigs repo, the `guix-codesign` step must be performed again in order for the guix attestation to be valid when compared against the attestations of non-codesigner builds. -Codesigner only: Commit the detached codesign payloads: +### Windows and macOS codesigners only: Commit the detached codesign payloads ```sh pushd ./bitcoin-detached-sigs @@ -178,16 +176,21 @@ git push the current branch and new tag popd ``` -Non-codesigners: wait for Windows/macOS detached signatures: +### Non-codesigners: wait for Windows and macOS detached signatures -- Once the Windows/macOS builds each have 3 matching signatures, they will be signed with their respective release keys. +- Once the Windows and macOS builds each have 3 matching signatures, they will be signed with their respective release keys. - Detached signatures will then be committed to the [bitcoin-detached-sigs](https://github.com/bitcoin-core/bitcoin-detached-sigs) repository, which can be combined with the unsigned apps to create signed binaries. -Create (and optionally verify) the codesigned outputs: +### Create the codesigned build outputs -- [Codesigning](/contrib/guix/README.md#codesigning) +- [Codesigning build outputs](/contrib/guix/README.md#codesigning-build-outputs) + +### Verify other builders' signatures to your own (optional) + +- [Add other builders keys to your gpg keyring, and/or refresh keys](/contrib/builder-keys/README.md) +- [Verifying build output attestations](/contrib/guix/README.md#verifying-build-output-attestations) -Commit your signature for the signed macOS/Windows binaries: +### Commit your codesigned signature to guix.sigs (for the signed macOS/Windows binaries) ```sh pushd ./guix.sigs @@ -197,7 +200,7 @@ git push # Assuming you can push to the guix.sigs tree popd ``` -### After 3 or more people have guix-built and their results match: +## After 3 or more people have guix-built and their results match Combine the `all.SHA256SUMS.asc` file from all signers into `SHA256SUMS.asc`: diff --git a/doc/tor.md b/doc/tor.md index 086e6747bf..08d031d084 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -16,9 +16,9 @@ configure Tor. ## How to see information about your Tor configuration via Bitcoin Core There are several ways to see your local onion address in Bitcoin Core: -- in the debug log (grep for "tor:" or "AddLocal") -- in the output of RPC `getnetworkinfo` in the "localaddresses" section -- in the output of the CLI `-netinfo` peer connections dashboard +- in the "Local addresses" output of CLI `-netinfo` +- in the "localaddresses" output of RPC `getnetworkinfo` +- in the debug log (grep for "AddLocal"; the Tor address ends in `.onion`) You may set the `-debug=tor` config logging option to have additional information in the debug log about your Tor configuration. @@ -27,6 +27,9 @@ CLI `-addrinfo` returns the number of addresses known to your node per network. This can be useful to see how many onion peers your node knows, e.g. for `-onlynet=onion`. +To fetch a number of onion addresses that your node knows, for example seven +addresses, use the `getnodeaddresses 7 onion` RPC. + ## 1. Run Bitcoin Core behind a Tor proxy The first step is running Bitcoin Core behind a Tor proxy. This will already anonymize all @@ -55,10 +58,10 @@ outgoing connections, but more is possible. -seednode=X SOCKS5. In Tor mode, such addresses can also be exchanged with other P2P nodes. - -onlynet=onion Make outgoing connections only to .onion addresses. Incoming - connections are not affected by this option. This option can be - specified multiple times to allow multiple network types, e.g. - onlynet=onion, onlynet=i2p. + -onlynet=onion Make automatic outbound connections only to .onion addresses. + Inbound and manual connections are not affected by this option. + It can be specified multiple times to allow multiple networks, + e.g. onlynet=onion, onlynet=i2p, onlynet=cjdns. In a typical situation, this suffices to run behind a Tor proxy: diff --git a/doc/tracing.md b/doc/tracing.md index 6ec6a6c218..b6e3b9263a 100644 --- a/doc/tracing.md +++ b/doc/tracing.md @@ -168,6 +168,49 @@ Arguments passed: 4. Value of the coin as `int64` 5. If the coin is a coinbase as `bool` +### Context `coin_selection` + +#### Tracepoint `coin_selection:selected_coins` + +Is called when `SelectCoins` completes. + +Arguments passed: +1. Wallet name as `pointer to C-style string` +2. Coin selection algorithm name as `pointer to C-style string` +3. Selection target value as `int64` +4. Calculated waste metric of the solution as `int64` +5. Total value of the selected inputs as `int64` + +#### Tracepoint `coin_selection:normal_create_tx_internal` + +Is called when the first `CreateTransactionInternal` completes. + +Arguments passed: +1. Wallet name as `pointer to C-style string` +2. Whether `CreateTransactionInternal` succeeded as `bool` +3. The expected transaction fee as an `int64` +4. The position of the change output as an `int32` + +#### Tracepoint `coin_selection:attempting_aps_create_tx` + +Is called when `CreateTransactionInternal` is called the second time for the optimistic +Avoid Partial Spends selection attempt. This is used to determine whether the next +tracepoints called are for the Avoid Partial Spends solution, or a different transaction. + +Arguments passed: +1. Wallet name as `pointer to C-style string` + +#### Tracepoint `coin_selection:aps_create_tx_internal` + +Is called when the second `CreateTransactionInternal` with Avoid Partial Spends enabled completes. + +Arguments passed: +1. Wallet name as `pointer to C-style string` +2. Whether the Avoid Partial Spends solution will be used as `bool` +3. Whether `CreateTransactionInternal` succeeded as` bool` +4. The expected transaction fee as an `int64` +5. The position of the change output as an `int32` + ## Adding tracepoints to Bitcoin Core To add a new tracepoint, `#include <util/trace.h>` in the compilation unit where diff --git a/share/examples/bitcoin.conf b/share/examples/bitcoin.conf index c5b79709c7..5bee4bf92e 100644 --- a/share/examples/bitcoin.conf +++ b/share/examples/bitcoin.conf @@ -1,191 +1 @@ -## -## bitcoin.conf configuration file. Lines beginning with # are comments. -## - -# Network-related settings: - -# Note that if you use testnet, signet or regtest, particularly with the options -# addnode, connect, port, bind, rpcport, rpcbind or wallet, you will also -# want to read "[Sections]" further down. - -# Run on the testnet network -#testnet=0 - -# Run on a signet network -#signet=0 - -# Run a regression test network -#regtest=0 - -# Connect via a SOCKS5 proxy -#proxy=127.0.0.1:9050 - -# Bind to given address and always listen on it. Use [host]:port notation for IPv6 -#bind=<addr> - -# Bind to given address and add permission flags to peers connecting to it. Use [host]:port notation for IPv6 -#whitebind=perm@<addr> - -############################################################## -## Quick Primer on addnode vs connect ## -## Let's say for instance you use addnode=4.2.2.4 ## -## addnode will connect you to and tell you about the ## -## nodes connected to 4.2.2.4. In addition it will tell ## -## the other nodes connected to it that you exist so ## -## they can connect to you. ## -## connect will not do the above when you 'connect' to it. ## -## It will *only* connect you to 4.2.2.4 and no one else.## -## ## -## So if you're behind a firewall, or have other problems ## -## finding nodes, add some using 'addnode'. ## -## ## -## If you want to stay private, use 'connect' to only ## -## connect to "trusted" nodes. ## -## ## -## If you run multiple nodes on a LAN, there's no need for ## -## all of them to open lots of connections. Instead ## -## 'connect' them all to one node that is port forwarded ## -## and has lots of connections. ## -## Thanks goes to [Noodle] on Freenode. ## -############################################################## - -# Use as many addnode= settings as you like to connect to specific peers -#addnode=69.164.218.197 -#addnode=10.0.0.2:8333 - -# Alternatively use as many connect= settings as you like to connect ONLY to specific peers -#connect=69.164.218.197 -#connect=10.0.0.1:8333 - -# Listening mode, enabled by default except when 'connect' is being used -#listen=1 - -# Port on which to listen for connections (default: 8333, testnet: 18333, signet: 38333, regtest: 18444) -#port= - -# Maximum number of inbound + outbound connections (default: 125). This option -# applies only if inbound connections are enabled; otherwise, the number of connections -# will not be more than 11: 8 full-relay connections, 2 block-relay-only ones, and -# occasionally 1 short-lived feeler or extra outbound block-relay-only connection. -# These limits do not apply to connections added manually with the -addnode -# configuration option or the addnode RPC, which have a separate limit of 8 connections. -#maxconnections= - -# Maximum upload bandwidth target in MiB per day (e.g. 'maxuploadtarget=1024' is 1 GiB per day). -# This limits the upload bandwidth for those with bandwidth limits. 0 = no limit (default: 0). -# -maxuploadtarget does not apply to peers with 'download' permission. -# For more information on reducing bandwidth utilization, see: doc/reduce-traffic.md. -#maxuploadtarget= - -# -# JSON-RPC options (for controlling a running Bitcoin/bitcoind process) -# - -# server=1 tells Bitcoin-Qt and bitcoind to accept JSON-RPC commands -#server=0 - -# Bind to given address to listen for JSON-RPC connections. -# Refer to the manpage or bitcoind -help for further details. -#rpcbind=<addr> - -# If no rpcpassword is set, rpc cookie auth is sought. The default `-rpccookiefile` name -# is .cookie and found in the `-datadir` being used for bitcoind. This option is typically used -# when the server and client are run as the same user. -# -# If not, you must set rpcuser and rpcpassword to secure the JSON-RPC API. -# -# The config option `rpcauth` can be added to server startup argument. It is set at initialization time -# using the output from the script in share/rpcauth/rpcauth.py after providing a username: -# -# ./share/rpcauth/rpcauth.py alice -# String to be appended to bitcoin.conf: -# rpcauth=alice:f7efda5c189b999524f151318c0c86$d5b51b3beffbc02b724e5d095828e0bc8b2456e9ac8757ae3211a5d9b16a22ae -# Your password: -# DONT_USE_THIS_YOU_WILL_GET_ROBBED_8ak1gI25KFTvjovL3gAM967mies3E= -# -# On client-side, you add the normal user/password pair to send commands: -#rpcuser=alice -#rpcpassword=DONT_USE_THIS_YOU_WILL_GET_ROBBED_8ak1gI25KFTvjovL3gAM967mies3E= -# -# You can even add multiple entries of these to the server conf file, and client can use any of them: -# rpcauth=bob:b2dd077cb54591a2f3139e69a897ac$4e71f08d48b4347cf8eff3815c0e25ae2e9a4340474079f55705f40574f4ec99 - -# How many seconds bitcoin will wait for a complete RPC HTTP request. -# after the HTTP connection is established. -#rpcclienttimeout=30 - -# By default, only RPC connections from localhost are allowed. -# Specify as many rpcallowip= settings as you like to allow connections from other hosts, -# either as a single IPv4/IPv6 or with a subnet specification. - -# NOTE: opening up the RPC port to hosts outside your local trusted network is NOT RECOMMENDED, -# because the rpcpassword is transmitted over the network unencrypted. - -# server=1 tells Bitcoin-Qt to accept JSON-RPC commands. -# it is also read by bitcoind to determine if RPC should be enabled -#rpcallowip=10.1.1.34/255.255.255.0 -#rpcallowip=1.2.3.4/24 -#rpcallowip=2001:db8:85a3:0:0:8a2e:370:7334/96 - -# Listen for RPC connections on this TCP port: -#rpcport=8332 - -# You can use Bitcoin or bitcoind to send commands to Bitcoin/bitcoind -# running on another host using this option: -#rpcconnect=127.0.0.1 - -# Wallet options - -# Specify where to find wallet, lockfile and logs. If not present, those files will be -# created as new. -#wallet=</path/to/dir> - -# Create transactions that have enough fees so they are likely to begin confirmation within n blocks (default: 6). -# This setting is over-ridden by the -paytxfee option. -#txconfirmtarget=n - -# Pay a transaction fee every time you send bitcoins. -#paytxfee=0.000x - -# Miscellaneous options - -# Pre-generate this many public/private key pairs, so wallet backups will be valid for -# both prior transactions and several dozen future transactions. -#keypool=100 - -# Maintain coinstats index used by the gettxoutsetinfo RPC (default: 0). -#coinstatsindex=1 - -# Enable pruning to reduce storage requirements by deleting old blocks. -# This mode is incompatible with -txindex and -coinstatsindex. -# 0 = default (no pruning). -# 1 = allows manual pruning via RPC. -# >=550 = target to stay under in MiB. -#prune=550 - -# User interface options - -# Start Bitcoin minimized -#min=1 - -# Minimize to the system tray -#minimizetotray=1 - -# [Sections] -# Most options apply to mainnet, testnet, signet and regtest. -# If you want to confine an option to just one network, you should add it in the -# relevant section below. -# EXCEPTIONS: The options addnode, connect, port, bind, rpcport, rpcbind and wallet -# only apply to mainnet unless they appear in the appropriate section below. - -# Options only for mainnet -[main] - -# Options only for testnet -[test] - -# Options only for signet -[signet] - -# Options only for regtest -[regtest] +# This is a placeholder file. Please follow the instructions in `contrib/devtools/README.md` to generate a bitcoin.conf file. diff --git a/src/.bear-tidy-config b/src/.bear-tidy-config new file mode 100644 index 0000000000..111ef6ee44 --- /dev/null +++ b/src/.bear-tidy-config @@ -0,0 +1,15 @@ +{ + "output": { + "content": { + "include_only_existing_source": true, + "paths_to_include": [], + "paths_to_exclude": [ + "src/leveldb" + ] + }, + "format": { + "command_as_array": true, + "drop_output_field": false + } + } +} diff --git a/src/.clang-tidy b/src/.clang-tidy index 27616ad072..b1f543f610 100644 --- a/src/.clang-tidy +++ b/src/.clang-tidy @@ -1,2 +1,11 @@ -Checks: '-*,bugprone-argument-comment' -WarningsAsErrors: bugprone-argument-comment +Checks: ' +-*, +bugprone-argument-comment, +modernize-use-nullptr, +readability-redundant-declaration, +' +WarningsAsErrors: ' +bugprone-argument-comment, +modernize-use-nullptr, +readability-redundant-declaration, +' diff --git a/src/Makefile.am b/src/Makefile.am index 417a611181..357e562c69 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -8,21 +8,30 @@ print-%: FORCE DIST_SUBDIRS = secp256k1 -AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) $(LTO_LDFLAGS) -AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) $(LTO_CXXFLAGS) -AM_CPPFLAGS = $(DEBUG_CPPFLAGS) $(HARDENED_CPPFLAGS) +AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) $(LTO_LDFLAGS) $(CORE_LDFLAGS) +AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) $(LTO_CXXFLAGS) $(CORE_CXXFLAGS) +AM_CPPFLAGS = $(DEBUG_CPPFLAGS) $(HARDENED_CPPFLAGS) $(CORE_CPPFLAGS) AM_LIBTOOLFLAGS = --preserve-dup-deps PTHREAD_FLAGS = $(PTHREAD_CFLAGS) $(PTHREAD_LIBS) EXTRA_LIBRARIES = +lib_LTLIBRARIES = +noinst_LTLIBRARIES = + +bin_PROGRAMS = +noinst_PROGRAMS = +TESTS = +BENCHMARKS = + BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) LIBBITCOIN_NODE=libbitcoin_node.a LIBBITCOIN_COMMON=libbitcoin_common.a LIBBITCOIN_CONSENSUS=libbitcoin_consensus.a LIBBITCOIN_CLI=libbitcoin_cli.a +LIBBITCOIN_KERNEL=libbitcoin_kernel.a LIBBITCOIN_UTIL=libbitcoin_util.a -LIBBITCOIN_CRYPTO_BASE=crypto/libbitcoin_crypto_base.a +LIBBITCOIN_CRYPTO_BASE=crypto/libbitcoin_crypto_base.la LIBBITCOINQT=qt/libbitcoinqt.a LIBSECP256K1=secp256k1/libsecp256k1.la @@ -32,28 +41,32 @@ endif if BUILD_BITCOIN_LIBS LIBBITCOINCONSENSUS=libbitcoinconsensus.la endif +if BUILD_BITCOIN_KERNEL_LIB +LIBBITCOINKERNEL=libbitcoinkernel.la +endif if ENABLE_WALLET LIBBITCOIN_WALLET=libbitcoin_wallet.a LIBBITCOIN_WALLET_TOOL=libbitcoin_wallet_tool.a endif -LIBBITCOIN_CRYPTO= $(LIBBITCOIN_CRYPTO_BASE) +LIBBITCOIN_CRYPTO = $(LIBBITCOIN_CRYPTO_BASE) if ENABLE_SSE41 -LIBBITCOIN_CRYPTO_SSE41 = crypto/libbitcoin_crypto_sse41.a +LIBBITCOIN_CRYPTO_SSE41 = crypto/libbitcoin_crypto_sse41.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_SSE41) endif if ENABLE_AVX2 -LIBBITCOIN_CRYPTO_AVX2 = crypto/libbitcoin_crypto_avx2.a +LIBBITCOIN_CRYPTO_AVX2 = crypto/libbitcoin_crypto_avx2.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_AVX2) endif if ENABLE_X86_SHANI -LIBBITCOIN_CRYPTO_X86_SHANI = crypto/libbitcoin_crypto_x86_shani.a +LIBBITCOIN_CRYPTO_X86_SHANI = crypto/libbitcoin_crypto_x86_shani.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_X86_SHANI) endif if ENABLE_ARM_SHANI -LIBBITCOIN_CRYPTO_ARM_SHANI = crypto/libbitcoin_crypto_arm_shani.a +LIBBITCOIN_CRYPTO_ARM_SHANI = crypto/libbitcoin_crypto_arm_shani.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_ARM_SHANI) endif +noinst_LTLIBRARIES += $(LIBBITCOIN_CRYPTO) $(LIBSECP256K1): $(wildcard secp256k1/src/*.h) $(wildcard secp256k1/src/*.c) $(wildcard secp256k1/include/*) $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) @@ -61,7 +74,6 @@ $(LIBSECP256K1): $(wildcard secp256k1/src/*.h) $(wildcard secp256k1/src/*.c) $(w # Make is not made aware of per-object dependencies to avoid limiting building parallelization # But to build the less dependent modules first, we manually select their order here: EXTRA_LIBRARIES += \ - $(LIBBITCOIN_CRYPTO) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_CONSENSUS) \ @@ -72,14 +84,6 @@ EXTRA_LIBRARIES += \ $(LIBBITCOIN_WALLET_TOOL) \ $(LIBBITCOIN_ZMQ) -lib_LTLIBRARIES = $(LIBBITCOINCONSENSUS) -noinst_LTLIBRARIES = - -bin_PROGRAMS = -noinst_PROGRAMS = -TESTS = -BENCHMARKS = - if BUILD_BITCOIND bin_PROGRAMS += bitcoind endif @@ -106,6 +110,10 @@ if BUILD_BITCOIN_UTIL bin_PROGRAMS += bitcoin-util endif +if BUILD_BITCOIN_CHAINSTATE + bin_PROGRAMS += bitcoin-chainstate +endif + .PHONY: FORCE check-symbols check-security # bitcoin core # BITCOIN_CORE_H = \ @@ -176,6 +184,7 @@ BITCOIN_CORE_H = \ net_types.h \ netaddress.h \ netbase.h \ + netgroup.h \ netmessagemaker.h \ node/blockstorage.h \ node/caches.h \ @@ -202,9 +211,11 @@ BITCOIN_CORE_H = \ psbt.h \ random.h \ randomenv.h \ + rest.h \ reverse_iterator.h \ rpc/blockchain.h \ rpc/client.h \ + rpc/mempool.h \ rpc/mining.h \ rpc/protocol.h \ rpc/rawtransaction_util.h \ @@ -216,6 +227,7 @@ BITCOIN_CORE_H = \ scheduler.h \ script/descriptor.h \ script/keyorigin.h \ + script/miniscript.h \ script/sigcache.h \ script/sign.h \ script/signingprovider.h \ @@ -263,6 +275,7 @@ BITCOIN_CORE_H = \ util/spanparsing.h \ util/string.h \ util/syscall_sandbox.h \ + util/syserror.h \ util/system.h \ util/thread.h \ util/threadnames.h \ @@ -345,6 +358,7 @@ libbitcoin_node_a_SOURCES = \ init.cpp \ mapport.cpp \ net.cpp \ + netgroup.cpp \ net_processing.cpp \ node/blockstorage.cpp \ node/caches.cpp \ @@ -366,12 +380,17 @@ libbitcoin_node_a_SOURCES = \ pow.cpp \ rest.cpp \ rpc/blockchain.cpp \ + rpc/fees.cpp \ + rpc/mempool.cpp \ rpc/mining.cpp \ - rpc/misc.cpp \ + rpc/node.cpp \ rpc/net.cpp \ + rpc/output_script.cpp \ rpc/rawtransaction.cpp \ rpc/server.cpp \ rpc/server_util.cpp \ + rpc/signmessage.cpp \ + rpc/txoutproof.cpp \ script/sigcache.cpp \ shutdown.cpp \ signet.cpp \ @@ -453,9 +472,16 @@ libbitcoin_wallet_tool_a_SOURCES = \ $(BITCOIN_CORE_H) # crypto primitives library -crypto_libbitcoin_crypto_base_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_base_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_base_a_SOURCES = \ +crypto_libbitcoin_crypto_base_la_CPPFLAGS = $(AM_CPPFLAGS) + +# Specify -static in both CXXFLAGS and LDFLAGS so libtool will only build a +# static version of this library. We don't need a dynamic version, and a dynamic +# version can't be used on windows anyway because the library doesn't currently +# export DLL symbols. +crypto_libbitcoin_crypto_base_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_base_la_LDFLAGS = $(AM_LDFLAGS) -static + +crypto_libbitcoin_crypto_base_la_SOURCES = \ crypto/aes.cpp \ crypto/aes.h \ crypto/chacha_poly_aead.h \ @@ -487,32 +513,44 @@ crypto_libbitcoin_crypto_base_a_SOURCES = \ crypto/siphash.h if USE_ASM -crypto_libbitcoin_crypto_base_a_SOURCES += crypto/sha256_sse4.cpp +crypto_libbitcoin_crypto_base_la_SOURCES += crypto/sha256_sse4.cpp endif -crypto_libbitcoin_crypto_sse41_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_sse41_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_sse41_a_CXXFLAGS += $(SSE41_CXXFLAGS) -crypto_libbitcoin_crypto_sse41_a_CPPFLAGS += -DENABLE_SSE41 -crypto_libbitcoin_crypto_sse41_a_SOURCES = crypto/sha256_sse41.cpp - -crypto_libbitcoin_crypto_avx2_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_avx2_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_avx2_a_CXXFLAGS += $(AVX2_CXXFLAGS) -crypto_libbitcoin_crypto_avx2_a_CPPFLAGS += -DENABLE_AVX2 -crypto_libbitcoin_crypto_avx2_a_SOURCES = crypto/sha256_avx2.cpp - -crypto_libbitcoin_crypto_x86_shani_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_x86_shani_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_x86_shani_a_CXXFLAGS += $(X86_SHANI_CXXFLAGS) -crypto_libbitcoin_crypto_x86_shani_a_CPPFLAGS += -DENABLE_X86_SHANI -crypto_libbitcoin_crypto_x86_shani_a_SOURCES = crypto/sha256_x86_shani.cpp - -crypto_libbitcoin_crypto_arm_shani_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_arm_shani_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_arm_shani_a_CXXFLAGS += $(ARM_SHANI_CXXFLAGS) -crypto_libbitcoin_crypto_arm_shani_a_CPPFLAGS += -DENABLE_ARM_SHANI -crypto_libbitcoin_crypto_arm_shani_a_SOURCES = crypto/sha256_arm_shani.cpp +# See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and +# CXXFLAGS above +crypto_libbitcoin_crypto_sse41_la_LDFLAGS = $(AM_LDFLAGS) -static +crypto_libbitcoin_crypto_sse41_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_sse41_la_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_sse41_la_CXXFLAGS += $(SSE41_CXXFLAGS) +crypto_libbitcoin_crypto_sse41_la_CPPFLAGS += -DENABLE_SSE41 +crypto_libbitcoin_crypto_sse41_la_SOURCES = crypto/sha256_sse41.cpp + +# See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and +# CXXFLAGS above +crypto_libbitcoin_crypto_avx2_la_LDFLAGS = $(AM_LDFLAGS) -static +crypto_libbitcoin_crypto_avx2_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_avx2_la_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_avx2_la_CXXFLAGS += $(AVX2_CXXFLAGS) +crypto_libbitcoin_crypto_avx2_la_CPPFLAGS += -DENABLE_AVX2 +crypto_libbitcoin_crypto_avx2_la_SOURCES = crypto/sha256_avx2.cpp + +# See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and +# CXXFLAGS above +crypto_libbitcoin_crypto_x86_shani_la_LDFLAGS = $(AM_LDFLAGS) -static +crypto_libbitcoin_crypto_x86_shani_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_x86_shani_la_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_x86_shani_la_CXXFLAGS += $(X86_SHANI_CXXFLAGS) +crypto_libbitcoin_crypto_x86_shani_la_CPPFLAGS += -DENABLE_X86_SHANI +crypto_libbitcoin_crypto_x86_shani_la_SOURCES = crypto/sha256_x86_shani.cpp + +# See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and +# CXXFLAGS above +crypto_libbitcoin_crypto_arm_shani_la_LDFLAGS = $(AM_LDFLAGS) -static +crypto_libbitcoin_crypto_arm_shani_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_arm_shani_la_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_arm_shani_la_CXXFLAGS += $(ARM_SHANI_CXXFLAGS) +crypto_libbitcoin_crypto_arm_shani_la_CPPFLAGS += -DENABLE_ARM_SHANI +crypto_libbitcoin_crypto_arm_shani_la_SOURCES = crypto/sha256_arm_shani.cpp # consensus: shared between all executables that validate any consensus rules. libbitcoin_consensus_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) @@ -583,6 +621,7 @@ libbitcoin_common_a_SOURCES = \ rpc/util.cpp \ scheduler.cpp \ script/descriptor.cpp \ + script/miniscript.cpp \ script/sign.cpp \ script/signingprovider.cpp \ script/standard.cpp \ @@ -599,7 +638,6 @@ libbitcoin_util_a_SOURCES = \ chainparamsbase.cpp \ clientversion.cpp \ compat/glibcxx_sanity.cpp \ - compat/strnlen.cpp \ fs.cpp \ interfaces/echo.cpp \ interfaces/handler.cpp \ @@ -614,11 +652,13 @@ libbitcoin_util_a_SOURCES = \ util/asmap.cpp \ util/bip32.cpp \ util/bytevectorhash.cpp \ + util/check.cpp \ util/error.cpp \ util/fees.cpp \ util/getuniquepath.cpp \ util/hasher.cpp \ util/sock.cpp \ + util/syserror.cpp \ util/system.cpp \ util/message.cpp \ util/moneystr.cpp \ @@ -671,7 +711,6 @@ bitcoin_bin_ldadd = \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ $(LIBSECP256K1) @@ -769,10 +808,142 @@ bitcoin_util_LDADD = \ $(LIBSECP256K1) # +# bitcoin-chainstate binary # +bitcoin_chainstate_SOURCES = bitcoin-chainstate.cpp +bitcoin_chainstate_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +bitcoin_chainstate_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) + +# $(LIBTOOL_APP_LDFLAGS) deliberately omitted here so that we can test linking +# bitcoin-chainstate against libbitcoinkernel as a shared or static library by +# setting --{en,dis}able-shared. +bitcoin_chainstate_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(PTHREAD_FLAGS) +bitcoin_chainstate_LDADD = $(LIBBITCOINKERNEL) +# + +# bitcoinkernel library # +if BUILD_BITCOIN_KERNEL_LIB +lib_LTLIBRARIES += $(LIBBITCOINKERNEL) + +libbitcoinkernel_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) $(PTHREAD_FLAGS) +libbitcoinkernel_la_LIBADD = $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) $(LIBSECP256K1) +libbitcoinkernel_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) + +# libbitcoinkernel requires default symbol visibility, explicitly specify that +# here so that things still work even when user configures with +# --enable-reduce-exports +# +# Note this is a quick hack that will be removed as we incrementally define what +# to export from the library. +libbitcoinkernel_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -fvisibility=default + +# TODO: For now, Specify -static in both CXXFLAGS and LDFLAGS when building for +# windows targets so libtool will only build a static version of this +# library. There are unresolved problems when building dll's for mingw-w64 +# and attempting to statically embed libstdc++, libpthread, etc. +if TARGET_WINDOWS +libbitcoinkernel_la_LDFLAGS += -static +libbitcoinkernel_la_CXXFLAGS += -static +endif + +# TODO: libbitcoinkernel is a work in progress consensus engine library, as more +# and more modules are decoupled from the consensus engine, this list will +# shrink to only those which are absolutely necessary. For example, things +# like index/*.cpp will be removed. +libbitcoinkernel_la_SOURCES = \ + kernel/bitcoinkernel.cpp \ + arith_uint256.cpp \ + blockfilter.cpp \ + chain.cpp \ + chainparamsbase.cpp \ + chainparams.cpp \ + clientversion.cpp \ + coins.cpp \ + compat/glibcxx_sanity.cpp \ + compressor.cpp \ + consensus/merkle.cpp \ + consensus/tx_check.cpp \ + consensus/tx_verify.cpp \ + core_read.cpp \ + dbwrapper.cpp \ + deploymentinfo.cpp \ + deploymentstatus.cpp \ + flatfile.cpp \ + fs.cpp \ + hash.cpp \ + index/base.cpp \ + index/blockfilterindex.cpp \ + index/coinstatsindex.cpp \ + init/common.cpp \ + key.cpp \ + logging.cpp \ + netaddress.cpp \ + node/blockstorage.cpp \ + node/chainstate.cpp \ + node/coinstats.cpp \ + node/ui_interface.cpp \ + policy/feerate.cpp \ + policy/fees.cpp \ + policy/packages.cpp \ + policy/policy.cpp \ + policy/rbf.cpp \ + policy/settings.cpp \ + pow.cpp \ + primitives/block.cpp \ + primitives/transaction.cpp \ + pubkey.cpp \ + random.cpp \ + randomenv.cpp \ + scheduler.cpp \ + script/interpreter.cpp \ + script/script.cpp \ + script/script_error.cpp \ + script/sigcache.cpp \ + script/standard.cpp \ + shutdown.cpp \ + signet.cpp \ + support/cleanse.cpp \ + support/lockedpool.cpp \ + sync.cpp \ + threadinterrupt.cpp \ + timedata.cpp \ + txdb.cpp \ + txmempool.cpp \ + uint256.cpp \ + util/asmap.cpp \ + util/bytevectorhash.cpp \ + util/check.cpp \ + util/getuniquepath.cpp \ + util/hasher.cpp \ + util/moneystr.cpp \ + util/rbf.cpp \ + util/serfloat.cpp \ + util/settings.cpp \ + util/strencodings.cpp \ + util/string.cpp \ + util/syscall_sandbox.cpp \ + util/syserror.cpp \ + util/system.cpp \ + util/thread.cpp \ + util/threadnames.cpp \ + util/time.cpp \ + util/tokenpipe.cpp \ + validation.cpp \ + validationinterface.cpp \ + versionbits.cpp \ + warnings.cpp + +# Required for obj/build.h to be generated first. +# More details: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html +libbitcoinkernel_la-clientversion.l$(OBJEXT): obj/build.h +endif # BUILD_BITCOIN_KERNEL_LIB +# + # bitcoinconsensus library # if BUILD_BITCOIN_LIBS +lib_LTLIBRARIES += $(LIBBITCOINCONSENSUS) + include_HEADERS = script/bitcoinconsensus.h -libbitcoinconsensus_la_SOURCES = support/cleanse.cpp $(crypto_libbitcoin_crypto_base_a_SOURCES) $(libbitcoin_consensus_a_SOURCES) +libbitcoinconsensus_la_SOURCES = support/cleanse.cpp $(crypto_libbitcoin_crypto_base_la_SOURCES) $(libbitcoin_consensus_a_SOURCES) libbitcoinconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) libbitcoinconsensus_la_LIBADD = $(LIBSECP256K1) diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 0bcce6ebe1..532f668668 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -13,53 +13,54 @@ GENERATED_BENCH_FILES = $(RAW_BENCH_FILES:.raw=.raw.h) bench_bench_bitcoin_SOURCES = \ $(RAW_BENCH_FILES) \ bench/addrman.cpp \ - bench/bench_bitcoin.cpp \ + bench/base58.cpp \ + bench/bech32.cpp \ bench/bench.cpp \ bench/bench.h \ + bench/bench_bitcoin.cpp \ bench/block_assemble.cpp \ + bench/ccoins_caching.cpp \ + bench/chacha20.cpp \ + bench/chacha_poly_aead.cpp \ bench/checkblock.cpp \ bench/checkqueue.cpp \ - bench/data.h \ + bench/crypto_hash.cpp \ bench/data.cpp \ + bench/data.h \ bench/duplicate_inputs.cpp \ bench/examples.cpp \ - bench/rollingbloom.cpp \ - bench/chacha20.cpp \ - bench/chacha_poly_aead.cpp \ - bench/crypto_hash.cpp \ - bench/ccoins_caching.cpp \ bench/gcs_filter.cpp \ bench/hashpadding.cpp \ - bench/merkle_root.cpp \ + bench/lockedpool.cpp \ + bench/logging.cpp \ bench/mempool_eviction.cpp \ bench/mempool_stress.cpp \ - bench/nanobench.h \ + bench/merkle_root.cpp \ bench/nanobench.cpp \ + bench/nanobench.h \ bench/peer_eviction.cpp \ + bench/poly1305.cpp \ + bench/prevector.cpp \ + bench/rollingbloom.cpp \ bench/rpc_blockchain.cpp \ bench/rpc_mempool.cpp \ + bench/strencodings.cpp \ bench/util_time.cpp \ - bench/verify_script.cpp \ - bench/base58.cpp \ - bench/bech32.cpp \ - bench/lockedpool.cpp \ - bench/poly1305.cpp \ - bench/prevector.cpp + bench/verify_script.cpp nodist_bench_bench_bitcoin_SOURCES = $(GENERATED_BENCH_FILES) bench_bench_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) -I$(builddir)/bench/ bench_bench_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) bench_bench_bitcoin_LDADD = \ + $(LIBTEST_UTIL) \ $(LIBBITCOIN_NODE) \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ - $(LIBTEST_UTIL) \ $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ $(LIBSECP256K1) \ $(LIBUNIVALUE) \ @@ -73,6 +74,7 @@ endif if ENABLE_WALLET bench_bench_bitcoin_SOURCES += bench/coin_selection.cpp bench_bench_bitcoin_SOURCES += bench/wallet_balance.cpp +bench_bench_bitcoin_SOURCES += bench/wallet_loading.cpp endif bench_bench_bitcoin_LDADD += $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) diff --git a/src/Makefile.crc32c.include b/src/Makefile.crc32c.include index 3cbe71792c..c4dd84991d 100644 --- a/src/Makefile.crc32c.include +++ b/src/Makefile.crc32c.include @@ -2,10 +2,9 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -LIBCRC32C_INT = crc32c/libcrc32c.a -LIBLEVELDB_SSE42_INT = leveldb/libleveldb_sse42.a +LIBCRC32C_INT = crc32c/libcrc32c.la -EXTRA_LIBRARIES += $(LIBCRC32C_INT) +noinst_LTLIBRARIES += $(LIBCRC32C_INT) LIBCRC32C = $(LIBCRC32C_INT) @@ -34,41 +33,49 @@ else CRC32C_CPPFLAGS_INT += -DBYTE_ORDER_BIG_ENDIAN=0 endif -crc32c_libcrc32c_a_CPPFLAGS = $(AM_CPPFLAGS) $(CRC32C_CPPFLAGS_INT) $(CRC32C_CPPFLAGS) -crc32c_libcrc32c_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) - -crc32c_libcrc32c_a_SOURCES = -crc32c_libcrc32c_a_SOURCES += crc32c/include/crc32c/crc32c.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_arm64.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_arm64_check.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_internal.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_prefetch.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_read_le.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_round_up.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_sse42_check.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_sse42.h - -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c.cc -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_portable.cc +crc32c_libcrc32c_la_CPPFLAGS = $(AM_CPPFLAGS) $(CRC32C_CPPFLAGS_INT) $(CRC32C_CPPFLAGS) + +# Specify -static in both CXXFLAGS and LDFLAGS so libtool will only build a +# static version of this library. We don't need a dynamic version, and a dynamic +# version can't be used on windows anyway because the library doesn't currently +# export DLL symbols. +crc32c_libcrc32c_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crc32c_libcrc32c_la_LDFLAGS = $(AM_LDFLAGS) -static + +crc32c_libcrc32c_la_SOURCES = +crc32c_libcrc32c_la_SOURCES += crc32c/include/crc32c/crc32c.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_arm64.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_arm64_check.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_internal.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_prefetch.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_read_le.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_round_up.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_sse42_check.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_sse42.h + +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c.cc +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_portable.cc if ENABLE_SSE42 -LIBCRC32C_SSE42_INT = crc32c/libcrc32c_sse42.a -EXTRA_LIBRARIES += $(LIBCRC32C_SSE42_INT) +LIBCRC32C_SSE42_INT = crc32c/libcrc32c_sse42.la +noinst_LTLIBRARIES += $(LIBCRC32C_SSE42_INT) LIBCRC32C += $(LIBCRC32C_SSE42_INT) -crc32c_libcrc32c_sse42_a_CPPFLAGS = $(crc32c_libcrc32c_a_CPPFLAGS) -crc32c_libcrc32c_sse42_a_CXXFLAGS = $(crc32c_libcrc32c_a_CXXFLAGS) $(SSE42_CXXFLAGS) +crc32c_libcrc32c_sse42_la_CPPFLAGS = $(crc32c_libcrc32c_la_CPPFLAGS) +crc32c_libcrc32c_sse42_la_CXXFLAGS = $(crc32c_libcrc32c_la_CXXFLAGS) $(SSE42_CXXFLAGS) +crc32c_libcrc32c_sse42_la_LDFLAGS = $(crc32c_libcrc32c_la_LDFLAGS) -crc32c_libcrc32c_sse42_a_SOURCES = crc32c/src/crc32c_sse42.cc +crc32c_libcrc32c_sse42_la_SOURCES = crc32c/src/crc32c_sse42.cc endif if ENABLE_ARM_CRC -LIBCRC32C_ARM_CRC_INT = crc32c/libcrc32c_arm_crc.a -EXTRA_LIBRARIES += $(LIBCRC32C_ARM_CRC_INT) +LIBCRC32C_ARM_CRC_INT = crc32c/libcrc32c_arm_crc.la +noinst_LTLIBRARIES += $(LIBCRC32C_ARM_CRC_INT) LIBCRC32C += $(LIBCRC32C_ARM_CRC_INT) -crc32c_libcrc32c_arm_crc_a_CPPFLAGS = $(crc32c_libcrc32c_a_CPPFLAGS) -crc32c_libcrc32c_arm_crc_a_CXXFLAGS = $(crc32c_libcrc32c_a_CXXFLAGS) $(ARM_CRC_CXXFLAGS) +crc32c_libcrc32c_arm_crc_la_CPPFLAGS = $(crc32c_libcrc32c_la_CPPFLAGS) +crc32c_libcrc32c_arm_crc_la_CXXFLAGS = $(crc32c_libcrc32c_la_CXXFLAGS) $(ARM_CRC_CXXFLAGS) +crc32c_libcrc32c_arm_crc_la_LDFLAGS = $(crc32c_libcrc32c_la_LDFLAGS) -crc32c_libcrc32c_arm_crc_a_SOURCES = crc32c/src/crc32c_arm64.cc +crc32c_libcrc32c_arm_crc_la_SOURCES = crc32c/src/crc32c_arm64.cc endif diff --git a/src/Makefile.leveldb.include b/src/Makefile.leveldb.include index 3bec92482a..066f8940c5 100644 --- a/src/Makefile.leveldb.include +++ b/src/Makefile.leveldb.include @@ -2,11 +2,11 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -LIBLEVELDB_INT = leveldb/libleveldb.a -LIBMEMENV_INT = leveldb/libmemenv.a +LIBLEVELDB_INT = leveldb/libleveldb.la +LIBMEMENV_INT = leveldb/libmemenv.la -EXTRA_LIBRARIES += $(LIBLEVELDB_INT) -EXTRA_LIBRARIES += $(LIBMEMENV_INT) +noinst_LTLIBRARIES += $(LIBLEVELDB_INT) +noinst_LTLIBRARIES += $(LIBMEMENV_INT) LIBLEVELDB = $(LIBLEVELDB_INT) $(LIBCRC32C) LIBMEMENV = $(LIBMEMENV_INT) @@ -37,111 +37,118 @@ else LEVELDB_CPPFLAGS_INT += -DLEVELDB_PLATFORM_POSIX endif -leveldb_libleveldb_a_CPPFLAGS = $(AM_CPPFLAGS) $(LEVELDB_CPPFLAGS_INT) $(LEVELDB_CPPFLAGS) -leveldb_libleveldb_a_CXXFLAGS = $(filter-out -Wconditional-uninitialized -Werror=conditional-uninitialized -Wsuggest-override -Werror=suggest-override, $(AM_CXXFLAGS)) $(PIE_FLAGS) +leveldb_libleveldb_la_CPPFLAGS = $(AM_CPPFLAGS) $(LEVELDB_CPPFLAGS_INT) $(LEVELDB_CPPFLAGS) -leveldb_libleveldb_a_SOURCES= -leveldb_libleveldb_a_SOURCES += leveldb/port/port_stdcxx.h -leveldb_libleveldb_a_SOURCES += leveldb/port/port.h -leveldb_libleveldb_a_SOURCES += leveldb/port/thread_annotations.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/db.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/options.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/comparator.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/filter_policy.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/slice.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/table_builder.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/env.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/export.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/c.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/iterator.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/cache.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/dumpfile.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/table.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/write_batch.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/status.h -leveldb_libleveldb_a_SOURCES += leveldb/db/log_format.h -leveldb_libleveldb_a_SOURCES += leveldb/db/memtable.h -leveldb_libleveldb_a_SOURCES += leveldb/db/version_set.h -leveldb_libleveldb_a_SOURCES += leveldb/db/write_batch_internal.h -leveldb_libleveldb_a_SOURCES += leveldb/db/filename.h -leveldb_libleveldb_a_SOURCES += leveldb/db/version_edit.h -leveldb_libleveldb_a_SOURCES += leveldb/db/dbformat.h -leveldb_libleveldb_a_SOURCES += leveldb/db/builder.h -leveldb_libleveldb_a_SOURCES += leveldb/db/log_writer.h -leveldb_libleveldb_a_SOURCES += leveldb/db/db_iter.h -leveldb_libleveldb_a_SOURCES += leveldb/db/skiplist.h -leveldb_libleveldb_a_SOURCES += leveldb/db/db_impl.h -leveldb_libleveldb_a_SOURCES += leveldb/db/table_cache.h -leveldb_libleveldb_a_SOURCES += leveldb/db/snapshot.h -leveldb_libleveldb_a_SOURCES += leveldb/db/log_reader.h -leveldb_libleveldb_a_SOURCES += leveldb/table/filter_block.h -leveldb_libleveldb_a_SOURCES += leveldb/table/block_builder.h -leveldb_libleveldb_a_SOURCES += leveldb/table/block.h -leveldb_libleveldb_a_SOURCES += leveldb/table/two_level_iterator.h -leveldb_libleveldb_a_SOURCES += leveldb/table/merger.h -leveldb_libleveldb_a_SOURCES += leveldb/table/format.h -leveldb_libleveldb_a_SOURCES += leveldb/table/iterator_wrapper.h -leveldb_libleveldb_a_SOURCES += leveldb/util/crc32c.h -leveldb_libleveldb_a_SOURCES += leveldb/util/env_posix_test_helper.h -leveldb_libleveldb_a_SOURCES += leveldb/util/env_windows_test_helper.h -leveldb_libleveldb_a_SOURCES += leveldb/util/arena.h -leveldb_libleveldb_a_SOURCES += leveldb/util/random.h -leveldb_libleveldb_a_SOURCES += leveldb/util/posix_logger.h -leveldb_libleveldb_a_SOURCES += leveldb/util/hash.h -leveldb_libleveldb_a_SOURCES += leveldb/util/histogram.h -leveldb_libleveldb_a_SOURCES += leveldb/util/coding.h -leveldb_libleveldb_a_SOURCES += leveldb/util/testutil.h -leveldb_libleveldb_a_SOURCES += leveldb/util/mutexlock.h -leveldb_libleveldb_a_SOURCES += leveldb/util/logging.h -leveldb_libleveldb_a_SOURCES += leveldb/util/no_destructor.h -leveldb_libleveldb_a_SOURCES += leveldb/util/testharness.h -leveldb_libleveldb_a_SOURCES += leveldb/util/windows_logger.h +# Specify -static in both CXXFLAGS and LDFLAGS so libtool will only build a +# static version of this library. We don't need a dynamic version, and a dynamic +# version can't be used on windows anyway because the library doesn't currently +# export DLL symbols. +leveldb_libleveldb_la_CXXFLAGS = $(filter-out -Wconditional-uninitialized -Werror=conditional-uninitialized -Wsuggest-override -Werror=suggest-override, $(AM_CXXFLAGS)) $(PIE_FLAGS) -static +leveldb_libleveldb_la_LDFLAGS = $(AM_LDFLAGS) -static -leveldb_libleveldb_a_SOURCES += leveldb/db/builder.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/c.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/dbformat.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/db_impl.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/db_iter.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/dumpfile.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/filename.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/log_reader.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/log_writer.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/memtable.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/repair.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/table_cache.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/version_edit.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/version_set.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/write_batch.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/block_builder.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/block.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/filter_block.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/format.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/iterator.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/merger.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/table_builder.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/table.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/two_level_iterator.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/arena.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/bloom.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/cache.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/coding.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/comparator.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/crc32c.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/env.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/filter_policy.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/hash.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/histogram.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/logging.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/options.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/status.cc +leveldb_libleveldb_la_SOURCES= +leveldb_libleveldb_la_SOURCES += leveldb/port/port_stdcxx.h +leveldb_libleveldb_la_SOURCES += leveldb/port/port.h +leveldb_libleveldb_la_SOURCES += leveldb/port/thread_annotations.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/db.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/options.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/comparator.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/filter_policy.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/slice.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/table_builder.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/env.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/export.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/c.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/iterator.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/cache.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/dumpfile.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/table.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/write_batch.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/status.h +leveldb_libleveldb_la_SOURCES += leveldb/db/log_format.h +leveldb_libleveldb_la_SOURCES += leveldb/db/memtable.h +leveldb_libleveldb_la_SOURCES += leveldb/db/version_set.h +leveldb_libleveldb_la_SOURCES += leveldb/db/write_batch_internal.h +leveldb_libleveldb_la_SOURCES += leveldb/db/filename.h +leveldb_libleveldb_la_SOURCES += leveldb/db/version_edit.h +leveldb_libleveldb_la_SOURCES += leveldb/db/dbformat.h +leveldb_libleveldb_la_SOURCES += leveldb/db/builder.h +leveldb_libleveldb_la_SOURCES += leveldb/db/log_writer.h +leveldb_libleveldb_la_SOURCES += leveldb/db/db_iter.h +leveldb_libleveldb_la_SOURCES += leveldb/db/skiplist.h +leveldb_libleveldb_la_SOURCES += leveldb/db/db_impl.h +leveldb_libleveldb_la_SOURCES += leveldb/db/table_cache.h +leveldb_libleveldb_la_SOURCES += leveldb/db/snapshot.h +leveldb_libleveldb_la_SOURCES += leveldb/db/log_reader.h +leveldb_libleveldb_la_SOURCES += leveldb/table/filter_block.h +leveldb_libleveldb_la_SOURCES += leveldb/table/block_builder.h +leveldb_libleveldb_la_SOURCES += leveldb/table/block.h +leveldb_libleveldb_la_SOURCES += leveldb/table/two_level_iterator.h +leveldb_libleveldb_la_SOURCES += leveldb/table/merger.h +leveldb_libleveldb_la_SOURCES += leveldb/table/format.h +leveldb_libleveldb_la_SOURCES += leveldb/table/iterator_wrapper.h +leveldb_libleveldb_la_SOURCES += leveldb/util/crc32c.h +leveldb_libleveldb_la_SOURCES += leveldb/util/env_posix_test_helper.h +leveldb_libleveldb_la_SOURCES += leveldb/util/env_windows_test_helper.h +leveldb_libleveldb_la_SOURCES += leveldb/util/arena.h +leveldb_libleveldb_la_SOURCES += leveldb/util/random.h +leveldb_libleveldb_la_SOURCES += leveldb/util/posix_logger.h +leveldb_libleveldb_la_SOURCES += leveldb/util/hash.h +leveldb_libleveldb_la_SOURCES += leveldb/util/histogram.h +leveldb_libleveldb_la_SOURCES += leveldb/util/coding.h +leveldb_libleveldb_la_SOURCES += leveldb/util/testutil.h +leveldb_libleveldb_la_SOURCES += leveldb/util/mutexlock.h +leveldb_libleveldb_la_SOURCES += leveldb/util/logging.h +leveldb_libleveldb_la_SOURCES += leveldb/util/no_destructor.h +leveldb_libleveldb_la_SOURCES += leveldb/util/testharness.h +leveldb_libleveldb_la_SOURCES += leveldb/util/windows_logger.h + +leveldb_libleveldb_la_SOURCES += leveldb/db/builder.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/c.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/dbformat.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/db_impl.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/db_iter.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/dumpfile.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/filename.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/log_reader.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/log_writer.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/memtable.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/repair.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/table_cache.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/version_edit.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/version_set.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/write_batch.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/block_builder.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/block.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/filter_block.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/format.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/iterator.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/merger.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/table_builder.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/table.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/two_level_iterator.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/arena.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/bloom.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/cache.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/coding.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/comparator.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/crc32c.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/env.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/filter_policy.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/hash.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/histogram.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/logging.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/options.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/status.cc if TARGET_WINDOWS -leveldb_libleveldb_a_SOURCES += leveldb/util/env_windows.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/env_windows.cc else -leveldb_libleveldb_a_SOURCES += leveldb/util/env_posix.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/env_posix.cc endif -leveldb_libmemenv_a_CPPFLAGS = $(leveldb_libleveldb_a_CPPFLAGS) -leveldb_libmemenv_a_CXXFLAGS = $(leveldb_libleveldb_a_CXXFLAGS) -leveldb_libmemenv_a_SOURCES = leveldb/helpers/memenv/memenv.cc -leveldb_libmemenv_a_SOURCES += leveldb/helpers/memenv/memenv.h +leveldb_libmemenv_la_CPPFLAGS = $(leveldb_libleveldb_la_CPPFLAGS) +leveldb_libmemenv_la_CXXFLAGS = $(leveldb_libleveldb_la_CXXFLAGS) +leveldb_libmemenv_la_LDFLAGS = $(leveldb_libleveldb_la_LDFLAGS) +leveldb_libmemenv_la_SOURCES = leveldb/helpers/memenv/memenv.cc +leveldb_libmemenv_la_SOURCES += leveldb/helpers/memenv/memenv.h diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 3491f07ee0..72037b3db2 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -330,7 +330,7 @@ endif if ENABLE_ZMQ bitcoin_qt_ldadd += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif -bitcoin_qt_ldadd += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ +bitcoin_qt_ldadd += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) \ $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(SQLITE_LIBS) bitcoin_qt_ldflags = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index 8e6fa2eb0d..fa822f2954 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -7,6 +7,7 @@ TESTS += qt/test/test_bitcoin-qt TEST_QT_MOC_CPP = \ qt/test/moc_apptests.cpp \ + qt/test/moc_optiontests.cpp \ qt/test/moc_rpcnestedtests.cpp \ qt/test/moc_uritests.cpp @@ -19,6 +20,7 @@ endif # ENABLE_WALLET TEST_QT_H = \ qt/test/addressbooktests.h \ qt/test/apptests.h \ + qt/test/optiontests.h \ qt/test/rpcnestedtests.h \ qt/test/uritests.h \ qt/test/util.h \ @@ -30,6 +32,7 @@ qt_test_test_bitcoin_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_ qt_test_test_bitcoin_qt_SOURCES = \ init/bitcoin-qt.cpp \ qt/test/apptests.cpp \ + qt/test/optiontests.cpp \ qt/test/rpcnestedtests.cpp \ qt/test/test_main.cpp \ qt/test/uritests.cpp \ @@ -52,7 +55,7 @@ if ENABLE_ZMQ qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(QT_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) \ + $(LIBMEMENV) $(QT_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) \ $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(SQLITE_LIBS) qt_test_test_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index e2e08468a7..02a3f9ae7d 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -47,7 +47,6 @@ FUZZ_SUITE_LD_COMMON = \ $(LIBBITCOIN_CLI) \ $(LIBUNIVALUE) \ $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ $(LIBSECP256K1) \ $(MINISKETCH_LIBS) \ @@ -94,6 +93,7 @@ BITCOIN_TESTS =\ test/fs_tests.cpp \ test/getarg_tests.cpp \ test/hash_tests.cpp \ + test/httpserver_tests.cpp \ test/i2p_tests.cpp \ test/interfaces_tests.cpp \ test/key_io_tests.cpp \ @@ -103,11 +103,13 @@ BITCOIN_TESTS =\ test/merkle_tests.cpp \ test/merkleblock_tests.cpp \ test/miner_tests.cpp \ + test/miniscript_tests.cpp \ test/minisketch_tests.cpp \ test/multisig_tests.cpp \ test/net_peer_eviction_tests.cpp \ test/net_tests.cpp \ test/netbase_tests.cpp \ + test/orphanage_tests.cpp \ test/pmt_tests.cpp \ test/policy_fee_tests.cpp \ test/policyestimator_tests.cpp \ @@ -115,12 +117,14 @@ BITCOIN_TESTS =\ test/prevector_tests.cpp \ test/raii_event_tests.cpp \ test/random_tests.cpp \ + test/rest_tests.cpp \ test/reverselock_tests.cpp \ test/rpc_tests.cpp \ test/sanity_tests.cpp \ test/scheduler_tests.cpp \ test/script_p2sh_tests.cpp \ test/script_parse_tests.cpp \ + test/script_segwit_tests.cpp \ test/script_standard_tests.cpp \ test/script_tests.cpp \ test/scriptnum10.h \ @@ -175,8 +179,11 @@ if USE_BDB BITCOIN_TESTS += wallet/test/db_tests.cpp endif -if USE_SQLITE FUZZ_WALLET_SRC = \ + wallet/test/fuzz/coinselection.cpp + +if USE_SQLITE +FUZZ_WALLET_SRC += \ wallet/test/fuzz/notifications.cpp endif # USE_SQLITE @@ -197,7 +204,7 @@ test_test_bitcoin_LDADD += $(LIBBITCOIN_WALLET) endif test_test_bitcoin_LDADD += $(LIBBITCOIN_NODE) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \ - $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) $(MINISKETCH_LIBS) + $(LIBLEVELDB) $(LIBMEMENV) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) $(MINISKETCH_LIBS) test_test_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_test_bitcoin_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) @@ -262,6 +269,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/locale.cpp \ test/fuzz/merkleblock.cpp \ test/fuzz/message.cpp \ + test/fuzz/miniscript_decode.cpp \ test/fuzz/minisketch.cpp \ test/fuzz/muhash.cpp \ test/fuzz/multiplication_overflow.cpp \ @@ -384,8 +392,19 @@ univalue_test_object_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) endif %.cpp.test: %.cpp - @echo Running tests: `cat $< | grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1` from $< - $(AM_V_at)$(TEST_BINARY) --catch_system_errors=no -l test_suite -t "`cat $< | grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1`" -- DEBUG_LOG_OUT > $<.log 2>&1 || (cat $<.log && false) + @echo Running tests: $$(\ + cat $< | \ + grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | \ + cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1\ + ) from $< + $(AM_V_at)export TEST_LOGFILE=$(abs_builddir)/$$(\ + echo $< | grep -E -o "(wallet/test/.*\.cpp|test/.*\.cpp)" | $(SED) -e s/\.cpp/.log/ \ + ) && \ + $(TEST_BINARY) --catch_system_errors=no -l test_suite -t "$$(\ + cat $< | \ + grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | \ + cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1\ + )" -- DEBUG_LOG_OUT > "$$TEST_LOGFILE" 2>&1 || (cat "$$TEST_LOGFILE" && false) %.json.h: %.json @$(MKDIR_P) $(@D) diff --git a/src/Makefile.test_fuzz.include b/src/Makefile.test_fuzz.include index 9574454fd2..8922dda3ad 100644 --- a/src/Makefile.test_fuzz.include +++ b/src/Makefile.test_fuzz.include @@ -18,8 +18,3 @@ libtest_fuzz_a_SOURCES = \ test/fuzz/fuzz.cpp \ test/fuzz/util.cpp \ $(TEST_FUZZ_H) - -LIBTEST_FUZZ += $(LIBBITCOIN_NODE) -LIBTEST_FUZZ += $(LIBBITCOIN_COMMON) -LIBTEST_FUZZ += $(LIBBITCOIN_UTIL) -LIBTEST_FUZZ += $(LIBBITCOIN_CRYPTO_BASE) diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index 92cb8a5ce6..9306bb6fcc 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -34,8 +34,3 @@ libtest_util_a_SOURCES = \ test/util/validation.cpp \ test/util/wallet.cpp \ $(TEST_UTIL_H) - -LIBTEST_UTIL += $(LIBBITCOIN_NODE) -LIBTEST_UTIL += $(LIBBITCOIN_COMMON) -LIBTEST_UTIL += $(LIBBITCOIN_UTIL) -LIBTEST_UTIL += $(LIBBITCOIN_CRYPTO_BASE) diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 0fa8f3c3da..b9eae292d7 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -13,6 +13,7 @@ #include <hash.h> #include <logging/timer.h> #include <netbase.h> +#include <netgroup.h> #include <random.h> #include <streams.h> #include <tinyformat.h> @@ -49,11 +50,11 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data { // Generate random temporary filename uint16_t randv = 0; - GetRandBytes((unsigned char*)&randv, sizeof(randv)); + GetRandBytes({(unsigned char*)&randv, sizeof(randv)}); std::string tmpfn = strprintf("%s.%04x", prefix, randv); // open temp output file, and associate with CAutoFile - fs::path pathTmp = gArgs.GetDataDirNet() / tmpfn; + fs::path pathTmp = gArgs.GetDataDirNet() / fs::u8path(tmpfn); FILE *file = fsbridge::fopen(pathTmp, "wb"); CAutoFile fileout(file, SER_DISK, version); if (fileout.IsNull()) { @@ -182,10 +183,10 @@ void ReadFromStream(AddrMan& addr, CDataStream& ssPeers) DeserializeDB(ssPeers, addr, false); } -std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman) +std::optional<bilingual_str> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman) { auto check_addrman = std::clamp<int32_t>(args.GetIntArg("-checkaddrman", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), 0, 1000000); - addrman = std::make_unique<AddrMan>(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); + addrman = std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); int64_t nStart = GetTimeMillis(); const auto path_addr{args.GetDataDirNet() / "peers.dat"}; @@ -194,7 +195,7 @@ std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const A LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman->size(), GetTimeMillis() - nStart); } catch (const DbNotFoundError&) { // Addrman can be in an inconsistent state after failure, reset it - addrman = std::make_unique<AddrMan>(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); + addrman = std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); LogPrintf("Creating peers.dat because the file was not found (%s)\n", fs::quoted(fs::PathToString(path_addr))); DumpPeerAddresses(args, *addrman); } catch (const InvalidAddrManVersionError&) { @@ -203,7 +204,7 @@ std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const A return strprintf(_("Failed to rename invalid peers.dat file. Please move or delete it and try again.")); } // Addrman can be in an inconsistent state after failure, reset it - addrman = std::make_unique<AddrMan>(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); + addrman = std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); LogPrintf("Creating new peers.dat because the file version was not compatible (%s). Original backed up to peers.dat.bak\n", fs::quoted(fs::PathToString(path_addr))); DumpPeerAddresses(args, *addrman); } catch (const std::exception& e) { diff --git a/src/addrdb.h b/src/addrdb.h index 4bdafb64e4..627ef3ac3c 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -17,6 +17,7 @@ class ArgsManager; class AddrMan; class CAddress; class CDataStream; +class NetGroupManager; struct bilingual_str; bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr); @@ -48,7 +49,7 @@ public: }; /** Returns an error string on failure */ -std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman); +std::optional<bilingual_str> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman); /** * Dump the anchor IP address database (anchors.dat) diff --git a/src/addrman.cpp b/src/addrman.cpp index f91a979934..f74729d47b 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -43,17 +43,17 @@ static constexpr size_t ADDRMAN_SET_TRIED_COLLISION_SIZE{10}; /** The maximum time we'll spend trying to resolve a tried table collision, in seconds */ static constexpr int64_t ADDRMAN_TEST_WINDOW{40*60}; // 40 minutes -int AddrInfo::GetTriedBucket(const uint256& nKey, const std::vector<bool>& asmap) const +int AddrInfo::GetTriedBucket(const uint256& nKey, const NetGroupManager& netgroupman) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetKey()).GetCheapHash(); - uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetCheapHash(); + uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << netgroupman.GetGroup(*this) << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetCheapHash(); return hash2 % ADDRMAN_TRIED_BUCKET_COUNT; } -int AddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std::vector<bool>& asmap) const +int AddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const NetGroupManager& netgroupman) const { - std::vector<unsigned char> vchSourceGroupKey = src.GetGroup(asmap); - uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << vchSourceGroupKey).GetCheapHash(); + std::vector<unsigned char> vchSourceGroupKey = netgroupman.GetGroup(src); + uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << netgroupman.GetGroup(*this) << vchSourceGroupKey).GetCheapHash(); uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << vchSourceGroupKey << (hash1 % ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP)).GetCheapHash(); return hash2 % ADDRMAN_NEW_BUCKET_COUNT; } @@ -99,11 +99,11 @@ double AddrInfo::GetChance(int64_t nNow) const return fChance; } -AddrManImpl::AddrManImpl(std::vector<bool>&& asmap, bool deterministic, int32_t consistency_check_ratio) +AddrManImpl::AddrManImpl(const NetGroupManager& netgroupman, bool deterministic, int32_t consistency_check_ratio) : insecure_rand{deterministic} , nKey{deterministic ? uint256{1} : insecure_rand.rand256()} , m_consistency_check_ratio{consistency_check_ratio} - , m_asmap{std::move(asmap)} + , m_netgroupman{netgroupman} { for (auto& bucket : vvNew) { for (auto& entry : bucket) { @@ -218,11 +218,7 @@ void AddrManImpl::Serialize(Stream& s_) const } // Store asmap checksum after bucket entries so that it // can be ignored by older clients for backward compatibility. - uint256 asmap_checksum; - if (m_asmap.size() != 0) { - asmap_checksum = SerializeHash(m_asmap); - } - s << asmap_checksum; + s << m_netgroupman.GetAsmapChecksum(); } template <typename Stream> @@ -246,12 +242,18 @@ void AddrManImpl::Unserialize(Stream& s_) uint8_t compat; s >> compat; + if (compat < INCOMPATIBILITY_BASE) { + throw std::ios_base::failure(strprintf( + "Corrupted addrman database: The compat value (%u) " + "is lower than the expected minimum value %u.", + compat, INCOMPATIBILITY_BASE)); + } const uint8_t lowest_compatible = compat - INCOMPATIBILITY_BASE; if (lowest_compatible > FILE_FORMAT) { throw InvalidAddrManVersionError(strprintf( "Unsupported format of addrman database: %u. It is compatible with formats >=%u, " "but the maximum supported by this version of %s is %u.", - uint8_t{format}, uint8_t{lowest_compatible}, PACKAGE_NAME, uint8_t{FILE_FORMAT})); + uint8_t{format}, lowest_compatible, PACKAGE_NAME, uint8_t{FILE_FORMAT})); } s >> nKey; @@ -292,7 +294,7 @@ void AddrManImpl::Unserialize(Stream& s_) for (int n = 0; n < nTried; n++) { AddrInfo info; s >> info; - int nKBucket = info.GetTriedBucket(nKey, m_asmap); + int nKBucket = info.GetTriedBucket(nKey, m_netgroupman); int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); if (info.IsValid() && vvTried[nKBucket][nKBucketPos] == -1) { @@ -329,10 +331,7 @@ void AddrManImpl::Unserialize(Stream& s_) // If the bucket count and asmap checksum haven't changed, then attempt // to restore the entries to the buckets/positions they were in before // serialization. - uint256 supplied_asmap_checksum; - if (m_asmap.size() != 0) { - supplied_asmap_checksum = SerializeHash(m_asmap); - } + uint256 supplied_asmap_checksum{m_netgroupman.GetAsmapChecksum()}; uint256 serialized_asmap_checksum; if (format >= Format::V2_ASMAP) { s >> serialized_asmap_checksum; @@ -365,7 +364,7 @@ void AddrManImpl::Unserialize(Stream& s_) } else { // In case the new table data cannot be used (bucket count wrong or new asmap), // try to give them a reference based on their primary source address. - bucket = info.GetNewBucket(nKey, m_asmap); + bucket = info.GetNewBucket(nKey, m_netgroupman); bucket_position = info.GetBucketPosition(nKey, true, bucket); if (vvNew[bucket][bucket_position] == -1) { vvNew[bucket][bucket_position] = entry_index; @@ -489,7 +488,7 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId) AssertLockHeld(cs); // remove the entry from all new buckets - const int start_bucket{info.GetNewBucket(nKey, m_asmap)}; + const int start_bucket{info.GetNewBucket(nKey, m_netgroupman)}; for (int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; ++n) { const int bucket{(start_bucket + n) % ADDRMAN_NEW_BUCKET_COUNT}; const int pos{info.GetBucketPosition(nKey, true, bucket)}; @@ -504,7 +503,7 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId) assert(info.nRefCount == 0); // which tried bucket to move the entry to - int nKBucket = info.GetTriedBucket(nKey, m_asmap); + int nKBucket = info.GetTriedBucket(nKey, m_netgroupman); int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); // first make space to add it (the existing tried entry there is moved to new, deleting whatever is there). @@ -520,7 +519,7 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId) nTried--; // find which new bucket it belongs to - int nUBucket = infoOld.GetNewBucket(nKey, m_asmap); + int nUBucket = infoOld.GetNewBucket(nKey, m_netgroupman); int nUBucketPos = infoOld.GetBucketPosition(nKey, true, nUBucket); ClearNew(nUBucket, nUBucketPos); assert(vvNew[nUBucket][nUBucketPos] == -1); @@ -588,7 +587,7 @@ bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, int64_ nNew++; } - int nUBucket = pinfo->GetNewBucket(nKey, source, m_asmap); + int nUBucket = pinfo->GetNewBucket(nKey, source, m_netgroupman); int nUBucketPos = pinfo->GetBucketPosition(nKey, true, nUBucket); bool fInsert = vvNew[nUBucket][nUBucketPos] == -1; if (vvNew[nUBucket][nUBucketPos] != nId) { @@ -604,7 +603,7 @@ bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, int64_ pinfo->nRefCount++; vvNew[nUBucket][nUBucketPos] = nId; LogPrint(BCLog::ADDRMAN, "Added %s mapped to AS%i to new[%i][%i]\n", - addr.ToString(), addr.GetMappedAS(m_asmap), nUBucket, nUBucketPos); + addr.ToString(), m_netgroupman.GetMappedAS(addr), nUBucket, nUBucketPos); } else { if (pinfo->nRefCount == 0) { Delete(nId); @@ -644,7 +643,7 @@ bool AddrManImpl::Good_(const CService& addr, bool test_before_evict, int64_t nT // which tried bucket to move the entry to - int tried_bucket = info.GetTriedBucket(nKey, m_asmap); + int tried_bucket = info.GetTriedBucket(nKey, m_netgroupman); int tried_bucket_pos = info.GetBucketPosition(nKey, false, tried_bucket); // Will moving this address into tried evict another entry? @@ -663,7 +662,7 @@ bool AddrManImpl::Good_(const CService& addr, bool test_before_evict, int64_t nT // move nId to the tried tables MakeTried(info, nId); LogPrint(BCLog::ADDRMAN, "Moved %s mapped to AS%i to tried[%i][%i]\n", - addr.ToString(), addr.GetMappedAS(m_asmap), tried_bucket, tried_bucket_pos); + addr.ToString(), m_netgroupman.GetMappedAS(addr), tried_bucket, tried_bucket_pos); return true; } } @@ -857,7 +856,7 @@ void AddrManImpl::ResolveCollisions_() AddrInfo& info_new = mapInfo[id_new]; // Which tried bucket to move the entry to. - int tried_bucket = info_new.GetTriedBucket(nKey, m_asmap); + int tried_bucket = info_new.GetTriedBucket(nKey, m_netgroupman); int tried_bucket_pos = info_new.GetBucketPosition(nKey, false, tried_bucket); if (!info_new.IsValid()) { // id_new may no longer map to a valid address erase_collision = true; @@ -923,7 +922,7 @@ std::pair<CAddress, int64_t> AddrManImpl::SelectTriedCollision_() const AddrInfo& newInfo = mapInfo[id_new]; // which tried bucket to move the entry to - int tried_bucket = newInfo.GetTriedBucket(nKey, m_asmap); + int tried_bucket = newInfo.GetTriedBucket(nKey, m_netgroupman); int tried_bucket_pos = newInfo.GetBucketPosition(nKey, false, tried_bucket); const AddrInfo& info_old = mapInfo[vvTried[tried_bucket][tried_bucket_pos]]; @@ -939,17 +938,17 @@ std::optional<AddressPosition> AddrManImpl::FindAddressEntry_(const CAddress& ad if (!addr_info) return std::nullopt; if(addr_info->fInTried) { - int bucket{addr_info->GetTriedBucket(nKey, m_asmap)}; - return AddressPosition(/*tried=*/true, - /*multiplicity=*/1, - /*bucket=*/bucket, - /*position=*/addr_info->GetBucketPosition(nKey, false, bucket)); + int bucket{addr_info->GetTriedBucket(nKey, m_netgroupman)}; + return AddressPosition(/*tried_in=*/true, + /*multiplicity_in=*/1, + /*bucket_in=*/bucket, + /*position_in=*/addr_info->GetBucketPosition(nKey, false, bucket)); } else { - int bucket{addr_info->GetNewBucket(nKey, m_asmap)}; - return AddressPosition(/*tried=*/false, - /*multiplicity=*/addr_info->nRefCount, - /*bucket=*/bucket, - /*position=*/addr_info->GetBucketPosition(nKey, true, bucket)); + int bucket{addr_info->GetNewBucket(nKey, m_netgroupman)}; + return AddressPosition(/*tried_in=*/false, + /*multiplicity_in=*/addr_info->nRefCount, + /*bucket_in=*/bucket, + /*position_in=*/addr_info->GetBucketPosition(nKey, true, bucket)); } } @@ -1020,7 +1019,7 @@ int AddrManImpl::CheckAddrman() const if (!setTried.count(vvTried[n][i])) return -11; const auto it{mapInfo.find(vvTried[n][i])}; - if (it == mapInfo.end() || it->second.GetTriedBucket(nKey, m_asmap) != n) { + if (it == mapInfo.end() || it->second.GetTriedBucket(nKey, m_netgroupman) != n) { return -17; } if (it->second.GetBucketPosition(nKey, false, n) != i) { @@ -1148,13 +1147,8 @@ std::optional<AddressPosition> AddrManImpl::FindAddressEntry(const CAddress& add return entry; } -const std::vector<bool>& AddrManImpl::GetAsmap() const -{ - return m_asmap; -} - -AddrMan::AddrMan(std::vector<bool> asmap, bool deterministic, int32_t consistency_check_ratio) - : m_impl(std::make_unique<AddrManImpl>(std::move(asmap), deterministic, consistency_check_ratio)) {} +AddrMan::AddrMan(const NetGroupManager& netgroupman, bool deterministic, int32_t consistency_check_ratio) + : m_impl(std::make_unique<AddrManImpl>(netgroupman, deterministic, consistency_check_ratio)) {} AddrMan::~AddrMan() = default; @@ -1229,11 +1223,6 @@ void AddrMan::SetServices(const CService& addr, ServiceFlags nServices) m_impl->SetServices(addr, nServices); } -const std::vector<bool>& AddrMan::GetAsmap() const -{ - return m_impl->GetAsmap(); -} - std::optional<AddressPosition> AddrMan::FindAddressEntry(const CAddress& addr) { return m_impl->FindAddressEntry(addr); diff --git a/src/addrman.h b/src/addrman.h index 472282833b..a0063e8a9c 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -7,6 +7,7 @@ #define BITCOIN_ADDRMAN_H #include <netaddress.h> +#include <netgroup.h> #include <protocol.h> #include <streams.h> #include <timedata.h> @@ -88,7 +89,7 @@ protected: const std::unique_ptr<AddrManImpl> m_impl; public: - explicit AddrMan(std::vector<bool> asmap, bool deterministic, int32_t consistency_check_ratio); + explicit AddrMan(const NetGroupManager& netgroupman, bool deterministic, int32_t consistency_check_ratio); ~AddrMan(); @@ -172,8 +173,6 @@ public: //! Update an entry's service bits. void SetServices(const CService& addr, ServiceFlags nServices); - const std::vector<bool>& GetAsmap() const; - /** Test-only function * Find the address record in AddrMan and return information about its * position. diff --git a/src/addrman_impl.h b/src/addrman_impl.h index 5e76f72342..9d98cdde54 100644 --- a/src/addrman_impl.h +++ b/src/addrman_impl.h @@ -76,15 +76,15 @@ public: } //! Calculate in which "tried" bucket this entry belongs - int GetTriedBucket(const uint256 &nKey, const std::vector<bool> &asmap) const; + int GetTriedBucket(const uint256& nKey, const NetGroupManager& netgroupman) const; //! Calculate in which "new" bucket this entry belongs, given a certain source - int GetNewBucket(const uint256 &nKey, const CNetAddr& src, const std::vector<bool> &asmap) const; + int GetNewBucket(const uint256& nKey, const CNetAddr& src, const NetGroupManager& netgroupman) const; //! Calculate in which "new" bucket this entry belongs, using its default source - int GetNewBucket(const uint256 &nKey, const std::vector<bool> &asmap) const + int GetNewBucket(const uint256& nKey, const NetGroupManager& netgroupman) const { - return GetNewBucket(nKey, source, asmap); + return GetNewBucket(nKey, source, netgroupman); } //! Calculate in which position of a bucket to store this entry. @@ -100,7 +100,7 @@ public: class AddrManImpl { public: - AddrManImpl(std::vector<bool>&& asmap, bool deterministic, int32_t consistency_check_ratio); + AddrManImpl(const NetGroupManager& netgroupman, bool deterministic, int32_t consistency_check_ratio); ~AddrManImpl(); @@ -140,8 +140,6 @@ public: std::optional<AddressPosition> FindAddressEntry(const CAddress& addr) EXCLUSIVE_LOCKS_REQUIRED(!cs); - const std::vector<bool>& GetAsmap() const; - friend class AddrManDeterministic; private: @@ -212,21 +210,8 @@ private: /** Perform consistency checks every m_consistency_check_ratio operations (if non-zero). */ const int32_t m_consistency_check_ratio; - // Compressed IP->ASN mapping, loaded from a file when a node starts. - // Should be always empty if no file was provided. - // This mapping is then used for bucketing nodes in Addrman. - // - // If asmap is provided, nodes will be bucketed by - // AS they belong to, in order to make impossible for a node - // to connect to several nodes hosted in a single AS. - // This is done in response to Erebus attack, but also to generally - // diversify the connections every node creates, - // especially useful when a large fraction of nodes - // operate under a couple of cloud providers. - // - // If a new asmap was provided, the existing records - // would be re-bucketed accordingly. - const std::vector<bool> m_asmap; + /** Reference to the netgroup manager. netgroupman must be constructed before addrman and destructed after. */ + const NetGroupManager& m_netgroupman; //! Find an entry. AddrInfo* Find(const CService& addr, int* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); diff --git a/src/arith_uint256.cpp b/src/arith_uint256.cpp index f7f62dfc68..e614102de3 100644 --- a/src/arith_uint256.cpp +++ b/src/arith_uint256.cpp @@ -146,13 +146,21 @@ double base_uint<BITS>::getdouble() const template <unsigned int BITS> std::string base_uint<BITS>::GetHex() const { - return ArithToUint256(*this).GetHex(); + base_blob<BITS> b; + for (int x = 0; x < this->WIDTH; ++x) { + WriteLE32(b.begin() + x*4, this->pn[x]); + } + return b.GetHex(); } template <unsigned int BITS> void base_uint<BITS>::SetHex(const char* psz) { - *this = UintToArith256(uint256S(psz)); + base_blob<BITS> b; + b.SetHex(psz); + for (int x = 0; x < this->WIDTH; ++x) { + this->pn[x] = ReadLE32(b.begin() + x*4); + } } template <unsigned int BITS> @@ -164,7 +172,7 @@ void base_uint<BITS>::SetHex(const std::string& str) template <unsigned int BITS> std::string base_uint<BITS>::ToString() const { - return (GetHex()); + return GetHex(); } template <unsigned int BITS> @@ -183,20 +191,7 @@ unsigned int base_uint<BITS>::bits() const } // Explicit instantiations for base_uint<256> -template base_uint<256>::base_uint(const std::string&); -template base_uint<256>& base_uint<256>::operator<<=(unsigned int); -template base_uint<256>& base_uint<256>::operator>>=(unsigned int); -template base_uint<256>& base_uint<256>::operator*=(uint32_t b32); -template base_uint<256>& base_uint<256>::operator*=(const base_uint<256>& b); -template base_uint<256>& base_uint<256>::operator/=(const base_uint<256>& b); -template int base_uint<256>::CompareTo(const base_uint<256>&) const; -template bool base_uint<256>::EqualTo(uint64_t) const; -template double base_uint<256>::getdouble() const; -template std::string base_uint<256>::GetHex() const; -template std::string base_uint<256>::ToString() const; -template void base_uint<256>::SetHex(const char*); -template void base_uint<256>::SetHex(const std::string&); -template unsigned int base_uint<256>::bits() const; +template class base_uint<256>; // This implementation directly uses shifts instead of going // through an intermediate MPI representation. diff --git a/src/arith_uint256.h b/src/arith_uint256.h index a0a0429c2a..b7b3b3a285 100644 --- a/src/arith_uint256.h +++ b/src/arith_uint256.h @@ -24,22 +24,19 @@ template<unsigned int BITS> class base_uint { protected: + static_assert(BITS / 32 > 0 && BITS % 32 == 0, "Template parameter BITS must be a positive multiple of 32."); static constexpr int WIDTH = BITS / 32; uint32_t pn[WIDTH]; public: base_uint() { - static_assert(BITS/32 > 0 && BITS%32 == 0, "Template parameter BITS must be a positive multiple of 32."); - for (int i = 0; i < WIDTH; i++) pn[i] = 0; } base_uint(const base_uint& b) { - static_assert(BITS/32 > 0 && BITS%32 == 0, "Template parameter BITS must be a positive multiple of 32."); - for (int i = 0; i < WIDTH; i++) pn[i] = b.pn[i]; } @@ -53,8 +50,6 @@ public: base_uint(uint64_t b) { - static_assert(BITS/32 > 0 && BITS%32 == 0, "Template parameter BITS must be a positive multiple of 32."); - pn[0] = (unsigned int)b; pn[1] = (unsigned int)(b >> 32); for (int i = 2; i < WIDTH; i++) @@ -284,4 +279,6 @@ public: uint256 ArithToUint256(const arith_uint256 &); arith_uint256 UintToArith256(const uint256 &); +extern template class base_uint<256>; + #endif // BITCOIN_ARITH_UINT256_H diff --git a/src/base58.cpp b/src/base58.cpp index dfa2e8db55..11c1ce7397 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -126,7 +126,7 @@ std::string EncodeBase58(Span<const unsigned char> input) bool DecodeBase58(const std::string& str, std::vector<unsigned char>& vchRet, int max_ret_len) { - if (!ValidAsCString(str)) { + if (!ContainsNoNUL(str)) { return false; } return DecodeBase58(str.c_str(), vchRet, max_ret_len); @@ -160,7 +160,7 @@ std::string EncodeBase58Check(Span<const unsigned char> input) bool DecodeBase58Check(const std::string& str, std::vector<unsigned char>& vchRet, int max_ret) { - if (!ValidAsCString(str)) { + if (!ContainsNoNUL(str)) { return false; } return DecodeBase58Check(str.c_str(), vchRet, max_ret); diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp index 3ca58b923e..76300f4db8 100644 --- a/src/bench/addrman.cpp +++ b/src/bench/addrman.cpp @@ -4,6 +4,7 @@ #include <addrman.h> #include <bench/bench.h> +#include <netgroup.h> #include <random.h> #include <util/check.h> #include <util/time.h> @@ -16,7 +17,7 @@ static constexpr size_t NUM_SOURCES = 64; static constexpr size_t NUM_ADDRESSES_PER_SOURCE = 256; -static const std::vector<bool> EMPTY_ASMAP; +static NetGroupManager EMPTY_NETGROUPMAN{std::vector<bool>()}; static constexpr uint32_t ADDRMAN_CONSISTENCY_CHECK_RATIO{0}; static std::vector<CAddress> g_sources; @@ -77,14 +78,14 @@ static void AddrManAdd(benchmark::Bench& bench) CreateAddresses(); bench.run([&] { - AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; + AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; AddAddressesToAddrMan(addrman); }); } static void AddrManSelect(benchmark::Bench& bench) { - AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; + AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; FillAddrMan(addrman); @@ -96,12 +97,12 @@ static void AddrManSelect(benchmark::Bench& bench) static void AddrManGetAddr(benchmark::Bench& bench) { - AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; + AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; FillAddrMan(addrman); bench.run([&] { - const auto& addresses = addrman.GetAddr(/* max_addresses */ 2500, /* max_pct */ 23, /* network */ std::nullopt); + const auto& addresses = addrman.GetAddr(/*max_addresses=*/2500, /*max_pct=*/23, /*network=*/std::nullopt); assert(addresses.size() > 0); }); } @@ -125,7 +126,7 @@ static void AddrManAddThenGood(benchmark::Bench& bench) // // This has some overhead (exactly the result of AddrManAdd benchmark), but that overhead is constant so improvements in // AddrMan::Good() will still be noticeable. - AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; + AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; AddAddressesToAddrMan(addrman); markSomeAsGood(addrman); diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index 3f8bff4bcf..d6f9c0f8b5 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -109,8 +109,8 @@ int main(int argc, char** argv) args.asymptote = parseAsymptote(argsman.GetArg("-asymptote", "")); args.is_list_only = argsman.GetBoolArg("-list", false); args.min_time = std::chrono::milliseconds(argsman.GetIntArg("-min_time", DEFAULT_MIN_TIME_MS)); - args.output_csv = fs::PathFromString(argsman.GetArg("-output_csv", "")); - args.output_json = fs::PathFromString(argsman.GetArg("-output_json", "")); + args.output_csv = argsman.GetPathArg("-output_csv"); + args.output_json = argsman.GetPathArg("-output_json"); args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER); benchmark::BenchRunner::RunAll(args); diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index d7b8c1badc..53591f8905 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -39,7 +39,10 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) { return true; } - void swap(PrevectorJob& x){p.swap(x.p);}; + void swap(PrevectorJob& x) noexcept + { + p.swap(x.p); + }; }; CCheckQueue<PrevectorJob> queue {QUEUE_BATCH_SIZE}; // The main thread should be counted to prevent thread oversubscription, and diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 609c592d20..de8ab9807c 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -13,7 +13,7 @@ using node::NodeContext; using wallet::AttemptSelection; -using wallet::CInputCoin; +using wallet::CHANGE_LOWER; using wallet::COutput; using wallet::CWallet; using wallet::CWalletTx; @@ -58,14 +58,22 @@ static void CoinSelection(benchmark::Bench& bench) // Create coins std::vector<COutput> coins; for (const auto& wtx : wtxs) { - coins.emplace_back(wallet, *wtx, 0 /* iIn */, 6 * 24 /* nDepthIn */, true /* spendable */, true /* solvable */, true /* safe */); + coins.emplace_back(COutPoint(wtx->GetHash(), 0), wtx->tx->vout.at(0), /*depth=*/6 * 24, GetTxSpendSize(wallet, *wtx, 0), /*spendable=*/true, /*solvable=*/true, /*safe=*/true, wtx->GetTxTime(), /*from_me=*/true); } const CoinEligibilityFilter filter_standard(1, 6, 0); - const CoinSelectionParams coin_selection_params(/* change_output_size= */ 34, - /* change_spend_size= */ 148, /* effective_feerate= */ CFeeRate(0), - /* long_term_feerate= */ CFeeRate(0), /* discard_feerate= */ CFeeRate(0), - /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); + FastRandomContext rand{}; + const CoinSelectionParams coin_selection_params{ + rand, + /*change_output_size=*/ 34, + /*change_spend_size=*/ 148, + /*min_change_target=*/ CHANGE_LOWER, + /*effective_feerate=*/ CFeeRate(0), + /*long_term_feerate=*/ CFeeRate(0), + /*discard_feerate=*/ CFeeRate(0), + /*tx_noinputs_size=*/ 0, + /*avoid_partial=*/ false, + }; bench.run([&] { auto result = AttemptSelection(wallet, 1003 * COIN, filter_standard, coins, coin_selection_params); assert(result); @@ -74,17 +82,15 @@ static void CoinSelection(benchmark::Bench& bench) }); } -typedef std::set<CInputCoin> CoinSet; - // Copied from src/wallet/test/coinselector_tests.cpp static void add_coin(const CAmount& nValue, int nInput, std::vector<OutputGroup>& set) { CMutableTransaction tx; tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; - CInputCoin coin(MakeTransactionRef(tx), nInput); + COutput output(COutPoint(tx.GetHash(), nInput), tx.vout.at(nInput), /*depth=*/ 0, /*input_bytes=*/ -1, /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, /*time=*/ 0, /*from_me=*/ true); set.emplace_back(); - set.back().Insert(coin, 0, true, 0, 0, false); + set.back().Insert(output, /*ancestors=*/ 0, /*descendants=*/ 0, /*positive_only=*/ false); } // Copied from src/wallet/test/coinselector_tests.cpp static CAmount make_hard_case(int utxos, std::vector<OutputGroup>& utxo_pool) diff --git a/src/bench/logging.cpp b/src/bench/logging.cpp new file mode 100644 index 0000000000..d28777df9e --- /dev/null +++ b/src/bench/logging.cpp @@ -0,0 +1,48 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <logging.h> +#include <test/util/setup_common.h> + + +static void Logging(benchmark::Bench& bench, const std::vector<const char*>& extra_args, const std::function<void()>& log) +{ + TestingSetup test_setup{ + CBaseChainParams::REGTEST, + extra_args, + }; + + bench.run([&] { log(); }); +} + +static void LoggingYoThreadNames(benchmark::Bench& bench) +{ + Logging(bench, {"-logthreadnames=1"}, [] { LogPrintf("%s\n", "test"); }); +} +static void LoggingNoThreadNames(benchmark::Bench& bench) +{ + Logging(bench, {"-logthreadnames=0"}, [] { LogPrintf("%s\n", "test"); }); +} +static void LoggingYoCategory(benchmark::Bench& bench) +{ + Logging(bench, {"-logthreadnames=0", "-debug=net"}, [] { LogPrint(BCLog::NET, "%s\n", "test"); }); +} +static void LoggingNoCategory(benchmark::Bench& bench) +{ + Logging(bench, {"-logthreadnames=0", "-debug=0"}, [] { LogPrint(BCLog::NET, "%s\n", "test"); }); +} +static void LoggingNoFile(benchmark::Bench& bench) +{ + Logging(bench, {"-nodebuglogfile", "-debug=1"}, [] { + LogPrintf("%s\n", "test"); + LogPrint(BCLog::NET, "%s\n", "test"); + }); +} + +BENCHMARK(LoggingYoThreadNames); +BENCHMARK(LoggingNoThreadNames); +BENCHMARK(LoggingYoCategory); +BENCHMARK(LoggingNoCategory); +BENCHMARK(LoggingNoFile); diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp index afa4618e1b..a58658c4f1 100644 --- a/src/bench/mempool_stress.cpp +++ b/src/bench/mempool_stress.cpp @@ -86,7 +86,7 @@ static void ComplexMemPool(benchmark::Bench& bench) if (bench.complexityN() > 1) { childTxs = static_cast<int>(bench.complexityN()); } - std::vector<CTransactionRef> ordered_coins = CreateOrderedCoins(det_rand, childTxs, /* min_ancestors */ 1); + std::vector<CTransactionRef> ordered_coins = CreateOrderedCoins(det_rand, childTxs, /*min_ancestors=*/1); const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(CBaseChainParams::MAIN); CTxMemPool pool; LOCK2(cs_main, pool.cs); @@ -103,7 +103,7 @@ static void MempoolCheck(benchmark::Bench& bench) { FastRandomContext det_rand{true}; const int childTxs = bench.complexityN() > 1 ? static_cast<int>(bench.complexityN()) : 2000; - const std::vector<CTransactionRef> ordered_coins = CreateOrderedCoins(det_rand, childTxs, /* min_ancestors */ 5); + const std::vector<CTransactionRef> ordered_coins = CreateOrderedCoins(det_rand, childTxs, /*min_ancestors=*/5); const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(CBaseChainParams::MAIN, {"-checkmempool=1"}); CTxMemPool pool; LOCK2(cs_main, pool.cs); @@ -111,7 +111,7 @@ static void MempoolCheck(benchmark::Bench& bench) for (auto& tx : ordered_coins) AddTx(tx, pool); bench.run([&]() NO_THREAD_SAFETY_ANALYSIS { - pool.check(coins_tip, /* spendheight */ 2); + pool.check(coins_tip, /*spendheight=*/2); }); } diff --git a/src/bench/rollingbloom.cpp b/src/bench/rollingbloom.cpp index 9f1679aa84..8f05e3bad0 100644 --- a/src/bench/rollingbloom.cpp +++ b/src/bench/rollingbloom.cpp @@ -5,6 +5,9 @@ #include <bench/bench.h> #include <common/bloom.h> +#include <crypto/common.h> + +#include <vector> static void RollingBloom(benchmark::Bench& bench) { @@ -13,16 +16,10 @@ static void RollingBloom(benchmark::Bench& bench) uint32_t count = 0; bench.run([&] { count++; - data[0] = count & 0xFF; - data[1] = (count >> 8) & 0xFF; - data[2] = (count >> 16) & 0xFF; - data[3] = (count >> 24) & 0xFF; + WriteLE32(data.data(), count); filter.insert(data); - data[0] = (count >> 24) & 0xFF; - data[1] = (count >> 16) & 0xFF; - data[2] = (count >> 8) & 0xFF; - data[3] = count & 0xFF; + WriteBE32(data.data(), count); filter.contains(data); }); } diff --git a/src/bench/rpc_blockchain.cpp b/src/bench/rpc_blockchain.cpp index 2143bcf950..e6fc8d21f4 100644 --- a/src/bench/rpc_blockchain.cpp +++ b/src/bench/rpc_blockchain.cpp @@ -40,7 +40,7 @@ static void BlockToJsonVerbose(benchmark::Bench& bench) { TestBlockAndIndex data; bench.run([&] { - auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, TxVerbosity::SHOW_DETAILS_AND_PREVOUT); + auto univalue = blockToJSON(data.testing_setup->m_node.chainman->m_blockman, data.block, &data.blockindex, &data.blockindex, TxVerbosity::SHOW_DETAILS_AND_PREVOUT); ankerl::nanobench::doNotOptimizeAway(univalue); }); } @@ -50,7 +50,7 @@ BENCHMARK(BlockToJsonVerbose); static void BlockToJsonVerboseWrite(benchmark::Bench& bench) { TestBlockAndIndex data; - auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, TxVerbosity::SHOW_DETAILS_AND_PREVOUT); + auto univalue = blockToJSON(data.testing_setup->m_node.chainman->m_blockman, data.block, &data.blockindex, &data.blockindex, TxVerbosity::SHOW_DETAILS_AND_PREVOUT); bench.run([&] { auto str = univalue.write(); ankerl::nanobench::doNotOptimizeAway(str); diff --git a/src/bench/rpc_mempool.cpp b/src/bench/rpc_mempool.cpp index 12dcff5844..6e322ba6aa 100644 --- a/src/bench/rpc_mempool.cpp +++ b/src/bench/rpc_mempool.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> -#include <rpc/blockchain.h> +#include <rpc/mempool.h> #include <txmempool.h> #include <univalue.h> @@ -29,11 +29,11 @@ static void RpcMempool(benchmark::Bench& bench) tx.vout[0].scriptPubKey = CScript() << OP_1 << OP_EQUAL; tx.vout[0].nValue = i; const CTransactionRef tx_r{MakeTransactionRef(tx)}; - AddTx(tx_r, /* fee */ i, pool); + AddTx(tx_r, /*fee=*/i, pool); } bench.run([&] { - (void)MempoolToJSON(pool, /*verbose*/ true); + (void)MempoolToJSON(pool, /*verbose=*/true); }); } diff --git a/src/bench/strencodings.cpp b/src/bench/strencodings.cpp new file mode 100644 index 0000000000..69b3a83cbf --- /dev/null +++ b/src/bench/strencodings.cpp @@ -0,0 +1,18 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <bench/data.h> +#include <util/strencodings.h> + +static void HexStrBench(benchmark::Bench& bench) +{ + auto const& data = benchmark::data::block413567; + bench.batch(data.size()).unit("byte").run([&] { + auto hex = HexStr(data); + ankerl::nanobench::doNotOptimizeAway(hex); + }); +} + +BENCHMARK(HexStrBench); diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index d4b8794c6d..a5dd2f3bb1 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -52,10 +52,10 @@ static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const b }); } -static void WalletBalanceDirty(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ true, /* add_mine */ true); } -static void WalletBalanceClean(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_mine */ true); } -static void WalletBalanceMine(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_mine */ true); } -static void WalletBalanceWatch(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_mine */ false); } +static void WalletBalanceDirty(benchmark::Bench& bench) { WalletBalance(bench, /*set_dirty=*/true, /*add_mine=*/true); } +static void WalletBalanceClean(benchmark::Bench& bench) { WalletBalance(bench, /*set_dirty=*/false, /*add_mine=*/true); } +static void WalletBalanceMine(benchmark::Bench& bench) { WalletBalance(bench, /*set_dirty=*/false, /*add_mine=*/true); } +static void WalletBalanceWatch(benchmark::Bench& bench) { WalletBalance(bench, /*set_dirty=*/false, /*add_mine=*/false); } BENCHMARK(WalletBalanceDirty); BENCHMARK(WalletBalanceClean); diff --git a/src/bench/wallet_loading.cpp b/src/bench/wallet_loading.cpp new file mode 100644 index 0000000000..38d3460001 --- /dev/null +++ b/src/bench/wallet_loading.cpp @@ -0,0 +1,83 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <interfaces/chain.h> +#include <node/context.h> +#include <test/util/mining.h> +#include <test/util/setup_common.h> +#include <test/util/wallet.h> +#include <util/translation.h> +#include <validationinterface.h> +#include <wallet/context.h> +#include <wallet/receive.h> +#include <wallet/wallet.h> + +#include <optional> + +using wallet::CWallet; +using wallet::DatabaseOptions; +using wallet::DatabaseStatus; +using wallet::ISMINE_SPENDABLE; +using wallet::MakeWalletDatabase; +using wallet::WALLET_FLAG_DESCRIPTORS; +using wallet::WalletContext; + +static const std::shared_ptr<CWallet> BenchLoadWallet(WalletContext& context, DatabaseOptions& options) +{ + DatabaseStatus status; + bilingual_str error; + std::vector<bilingual_str> warnings; + auto database = MakeWalletDatabase("", options, status, error); + assert(database); + auto wallet = CWallet::Create(context, "", std::move(database), options.create_flags, error, warnings); + NotifyWalletLoaded(context, wallet); + if (context.chain) { + wallet->postInitProcess(); + } + return wallet; +} + +static void BenchUnloadWallet(std::shared_ptr<CWallet>&& wallet) +{ + SyncWithValidationInterfaceQueue(); + wallet->m_chain_notifications_handler.reset(); + UnloadWallet(std::move(wallet)); +} + +static void WalletLoading(benchmark::Bench& bench, bool legacy_wallet) +{ + const auto test_setup = MakeNoLogFileContext<TestingSetup>(); + + WalletContext context; + context.args = &test_setup->m_args; + context.chain = test_setup->m_node.chain.get(); + + // Setup the wallet + // Loading the wallet will also create it + DatabaseOptions options; + if (!legacy_wallet) options.create_flags = WALLET_FLAG_DESCRIPTORS; + auto wallet = BenchLoadWallet(context, options); + + // Generate a bunch of transactions and addresses to put into the wallet + for (int i = 0; i < 5000; ++i) { + generatetoaddress(test_setup->m_node, getnewaddress(*wallet)); + } + + // reload the wallet for the actual benchmark + BenchUnloadWallet(std::move(wallet)); + + bench.minEpochIterations(10).run([&] { + wallet = BenchLoadWallet(context, options); + + // Cleanup + BenchUnloadWallet(std::move(wallet)); + }); +} + +static void WalletLoadingLegacy(benchmark::Bench& bench) { WalletLoading(bench, /*legacy_wallet=*/true); } +static void WalletLoadingDescriptors(benchmark::Bench& bench) { WalletLoading(bench, /*legacy_wallet=*/false); } + +BENCHMARK(WalletLoadingLegacy); +BENCHMARK(WalletLoadingDescriptors); diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp new file mode 100644 index 0000000000..e5b6875a36 --- /dev/null +++ b/src/bitcoin-chainstate.cpp @@ -0,0 +1,258 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +// The bitcoin-chainstate executable serves to surface the dependencies required +// by a program wishing to use Bitcoin Core's consensus engine as it is right +// now. +// +// DEVELOPER NOTE: Since this is a "demo-only", experimental, etc. executable, +// it may diverge from Bitcoin Core's coding style. +// +// It is part of the libbitcoinkernel project. + +#include <chainparams.h> +#include <consensus/validation.h> +#include <core_io.h> +#include <init/common.h> +#include <node/blockstorage.h> +#include <node/chainstate.h> +#include <scheduler.h> +#include <script/sigcache.h> +#include <util/system.h> +#include <util/thread.h> +#include <validation.h> +#include <validationinterface.h> + +#include <filesystem> +#include <functional> +#include <iosfwd> + +int main(int argc, char* argv[]) +{ + // SETUP: Argument parsing and handling + if (argc != 2) { + std::cerr + << "Usage: " << argv[0] << " DATADIR" << std::endl + << "Display DATADIR information, and process hex-encoded blocks on standard input." << std::endl + << std::endl + << "IMPORTANT: THIS EXECUTABLE IS EXPERIMENTAL, FOR TESTING ONLY, AND EXPECTED TO" << std::endl + << " BREAK IN FUTURE VERSIONS. DO NOT USE ON YOUR ACTUAL DATADIR." << std::endl; + return 1; + } + std::filesystem::path abs_datadir = std::filesystem::absolute(argv[1]); + std::filesystem::create_directories(abs_datadir); + gArgs.ForceSetArg("-datadir", abs_datadir.string()); + + + // SETUP: Misc Globals + SelectParams(CBaseChainParams::MAIN); + const CChainParams& chainparams = Params(); + + init::SetGlobals(); // ECC_Start, etc. + + // Necessary for CheckInputScripts (eventually called by ProcessNewBlock), + // which will try the script cache first and fall back to actually + // performing the check with the signature cache. + InitSignatureCache(); + InitScriptExecutionCache(); + + + // SETUP: Scheduling and Background Signals + CScheduler scheduler{}; + // Start the lightweight task scheduler thread + scheduler.m_service_thread = std::thread(util::TraceThread, "scheduler", [&] { scheduler.serviceQueue(); }); + + // Gather some entropy once per minute. + scheduler.scheduleEvery(RandAddPeriodic, std::chrono::minutes{1}); + + GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); + + + // SETUP: Chainstate + ChainstateManager chainman; + + auto rv = node::LoadChainstate(false, + std::ref(chainman), + nullptr, + false, + chainparams.GetConsensus(), + false, + 2 << 20, + 2 << 22, + (450 << 20) - (2 << 20) - (2 << 22), + false, + false, + []() { return false; }); + if (rv.has_value()) { + std::cerr << "Failed to load Chain state from your datadir." << std::endl; + goto epilogue; + } else { + auto maybe_verify_error = node::VerifyLoadedChainstate(std::ref(chainman), + false, + false, + chainparams.GetConsensus(), + DEFAULT_CHECKBLOCKS, + DEFAULT_CHECKLEVEL, + /*get_unix_time_seconds=*/static_cast<int64_t (*)()>(GetTime)); + if (maybe_verify_error.has_value()) { + std::cerr << "Failed to verify loaded Chain state from your datadir." << std::endl; + goto epilogue; + } + } + + for (CChainState* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) { + BlockValidationState state; + if (!chainstate->ActivateBestChain(state, nullptr)) { + std::cerr << "Failed to connect best block (" << state.ToString() << ")" << std::endl; + goto epilogue; + } + } + + // Main program logic starts here + std::cout + << "Hello! I'm going to print out some information about your datadir." << std::endl + << "\t" << "Path: " << gArgs.GetDataDirNet() << std::endl + << "\t" << "Reindexing: " << std::boolalpha << node::fReindex.load() << std::noboolalpha << std::endl + << "\t" << "Snapshot Active: " << std::boolalpha << chainman.IsSnapshotActive() << std::noboolalpha << std::endl + << "\t" << "Active Height: " << chainman.ActiveHeight() << std::endl + << "\t" << "Active IBD: " << std::boolalpha << chainman.ActiveChainstate().IsInitialBlockDownload() << std::noboolalpha << std::endl; + { + CBlockIndex* tip = chainman.ActiveTip(); + if (tip) { + std::cout << "\t" << tip->ToString() << std::endl; + } + } + + for (std::string line; std::getline(std::cin, line);) { + if (line.empty()) { + std::cerr << "Empty line found" << std::endl; + break; + } + + std::shared_ptr<CBlock> blockptr = std::make_shared<CBlock>(); + CBlock& block = *blockptr; + + if (!DecodeHexBlk(block, line)) { + std::cerr << "Block decode failed" << std::endl; + break; + } + + if (block.vtx.empty() || !block.vtx[0]->IsCoinBase()) { + std::cerr << "Block does not start with a coinbase" << std::endl; + break; + } + + uint256 hash = block.GetHash(); + { + LOCK(cs_main); + const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(hash); + if (pindex) { + if (pindex->IsValid(BLOCK_VALID_SCRIPTS)) { + std::cerr << "duplicate" << std::endl; + break; + } + if (pindex->nStatus & BLOCK_FAILED_MASK) { + std::cerr << "duplicate-invalid" << std::endl; + break; + } + } + } + + { + LOCK(cs_main); + const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock); + if (pindex) { + UpdateUncommittedBlockStructures(block, pindex, chainparams.GetConsensus()); + } + } + + // Adapted from rpc/mining.cpp + class submitblock_StateCatcher final : public CValidationInterface + { + public: + uint256 hash; + bool found; + BlockValidationState state; + + explicit submitblock_StateCatcher(const uint256& hashIn) : hash(hashIn), found(false), state() {} + + protected: + void BlockChecked(const CBlock& block, const BlockValidationState& stateIn) override + { + if (block.GetHash() != hash) + return; + found = true; + state = stateIn; + } + }; + + bool new_block; + auto sc = std::make_shared<submitblock_StateCatcher>(block.GetHash()); + RegisterSharedValidationInterface(sc); + bool accepted = chainman.ProcessNewBlock(chainparams, blockptr, /*force_processing=*/true, /*new_block=*/&new_block); + UnregisterSharedValidationInterface(sc); + if (!new_block && accepted) { + std::cerr << "duplicate" << std::endl; + break; + } + if (!sc->found) { + std::cerr << "inconclusive" << std::endl; + break; + } + std::cout << sc->state.ToString() << std::endl; + switch (sc->state.GetResult()) { + case BlockValidationResult::BLOCK_RESULT_UNSET: + std::cerr << "initial value. Block has not yet been rejected" << std::endl; + break; + case BlockValidationResult::BLOCK_CONSENSUS: + std::cerr << "invalid by consensus rules (excluding any below reasons)" << std::endl; + break; + case BlockValidationResult::BLOCK_RECENT_CONSENSUS_CHANGE: + std::cerr << "Invalid by a change to consensus rules more recent than SegWit." << std::endl; + break; + case BlockValidationResult::BLOCK_CACHED_INVALID: + std::cerr << "this block was cached as being invalid and we didn't store the reason why" << std::endl; + break; + case BlockValidationResult::BLOCK_INVALID_HEADER: + std::cerr << "invalid proof of work or time too old" << std::endl; + break; + case BlockValidationResult::BLOCK_MUTATED: + std::cerr << "the block's data didn't match the data committed to by the PoW" << std::endl; + break; + case BlockValidationResult::BLOCK_MISSING_PREV: + std::cerr << "We don't have the previous block the checked one is built on" << std::endl; + break; + case BlockValidationResult::BLOCK_INVALID_PREV: + std::cerr << "A block this one builds on is invalid" << std::endl; + break; + case BlockValidationResult::BLOCK_TIME_FUTURE: + std::cerr << "block timestamp was > 2 hours in the future (or our clock is bad)" << std::endl; + break; + case BlockValidationResult::BLOCK_CHECKPOINT: + std::cerr << "the block failed to meet one of our checkpoints" << std::endl; + break; + } + } + +epilogue: + // Without this precise shutdown sequence, there will be a lot of nullptr + // dereferencing and UB. + scheduler.stop(); + if (chainman.m_load_block.joinable()) chainman.m_load_block.join(); + StopScriptCheckWorkerThreads(); + + GetMainSignals().FlushBackgroundCallbacks(); + { + LOCK(cs_main); + for (CChainState* chainstate : chainman.GetAll()) { + if (chainstate->CanFlushToDisk()) { + chainstate->ForceFlushStateToDisk(); + chainstate->ResetCoinsViews(); + } + } + } + GetMainSignals().UnregisterBackgroundSignalScheduler(); + + init::UnsetGlobals(); +} diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 5523fff3b2..dea46693bc 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -184,7 +184,6 @@ struct HTTPReply static std::string http_errorstring(int code) { switch(code) { -#if LIBEVENT_VERSION_NUMBER >= 0x02010300 case EVREQ_HTTP_TIMEOUT: return "timeout reached"; case EVREQ_HTTP_EOF: @@ -197,7 +196,6 @@ static std::string http_errorstring(int code) return "request was canceled"; case EVREQ_HTTP_DATA_TOO_LONG: return "response body is larger than allowed"; -#endif default: return "unknown"; } @@ -228,13 +226,11 @@ static void http_request_done(struct evhttp_request *req, void *ctx) } } -#if LIBEVENT_VERSION_NUMBER >= 0x02010300 static void http_error_cb(enum evhttp_request_error err, void *ctx) { HTTPReply *reply = static_cast<HTTPReply*>(ctx); reply->error = err; } -#endif /** Class that handles the conversion from a command-line to a JSON-RPC request, * as well as converting back to a JSON object that can be shown as result. @@ -745,11 +741,11 @@ static UniValue CallRPC(BaseRequestHandler* rh, const std::string& strMethod, co HTTPReply response; raii_evhttp_request req = obtain_evhttp_request(http_request_done, (void*)&response); - if (req == nullptr) + if (req == nullptr) { throw std::runtime_error("create http request failed"); -#if LIBEVENT_VERSION_NUMBER >= 0x02010300 + } + evhttp_request_set_error_cb(req.get(), http_error_cb); -#endif // Get credentials std::string strRPCUserColonPass; @@ -1059,7 +1055,9 @@ static void ParseGetInfoResult(UniValue& result) result_string += "\n"; } - result_string += strprintf("%sWarnings:%s %s", YELLOW, RESET, result["warnings"].getValStr()); + const std::string warnings{result["warnings"].getValStr()}; + result_string += strprintf("%sWarnings:%s %s", YELLOW, RESET, warnings.empty() ? "(none)" : warnings); + result.setStr(result_string); } diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index b297081cab..580cc5839a 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -31,8 +31,6 @@ #include <memory> #include <stdio.h> -#include <boost/algorithm/string.hpp> - static bool fCreateBlank; static std::map<std::string,UniValue> registers; static const int CONTINUE_EXECUTION=-1; @@ -242,7 +240,7 @@ static void MutateTxRBFOptIn(CMutableTransaction& tx, const std::string& strInId template <typename T> static T TrimAndParse(const std::string& int_str, const std::string& err) { - const auto parsed{ToIntegral<T>(TrimString(int_str))}; + const auto parsed{ToIntegral<T>(TrimStringView(int_str))}; if (!parsed.has_value()) { throw std::runtime_error(err + " '" + int_str + "'"); } @@ -251,8 +249,7 @@ static T TrimAndParse(const std::string& int_str, const std::string& err) static void MutateTxAddInput(CMutableTransaction& tx, const std::string& strInput) { - std::vector<std::string> vStrInputParts; - boost::split(vStrInputParts, strInput, boost::is_any_of(":")); + std::vector<std::string> vStrInputParts = SplitString(strInput, ':'); // separate TXID:VOUT in string if (vStrInputParts.size()<2) @@ -287,8 +284,7 @@ static void MutateTxAddInput(CMutableTransaction& tx, const std::string& strInpu static void MutateTxAddOutAddr(CMutableTransaction& tx, const std::string& strInput) { // Separate into VALUE:ADDRESS - std::vector<std::string> vStrInputParts; - boost::split(vStrInputParts, strInput, boost::is_any_of(":")); + std::vector<std::string> vStrInputParts = SplitString(strInput, ':'); if (vStrInputParts.size() != 2) throw std::runtime_error("TX output missing or too many separators"); @@ -312,8 +308,7 @@ static void MutateTxAddOutAddr(CMutableTransaction& tx, const std::string& strIn static void MutateTxAddOutPubKey(CMutableTransaction& tx, const std::string& strInput) { // Separate into VALUE:PUBKEY[:FLAGS] - std::vector<std::string> vStrInputParts; - boost::split(vStrInputParts, strInput, boost::is_any_of(":")); + std::vector<std::string> vStrInputParts = SplitString(strInput, ':'); if (vStrInputParts.size() < 2 || vStrInputParts.size() > 3) throw std::runtime_error("TX output missing or too many separators"); @@ -356,8 +351,7 @@ static void MutateTxAddOutPubKey(CMutableTransaction& tx, const std::string& str static void MutateTxAddOutMultiSig(CMutableTransaction& tx, const std::string& strInput) { // Separate into VALUE:REQUIRED:NUMKEYS:PUBKEY1:PUBKEY2:....[:FLAGS] - std::vector<std::string> vStrInputParts; - boost::split(vStrInputParts, strInput, boost::is_any_of(":")); + std::vector<std::string> vStrInputParts = SplitString(strInput, ':'); // Check that there are enough parameters if (vStrInputParts.size()<3) @@ -460,8 +454,7 @@ static void MutateTxAddOutData(CMutableTransaction& tx, const std::string& strIn static void MutateTxAddOutScript(CMutableTransaction& tx, const std::string& strInput) { // separate VALUE:SCRIPT[:FLAGS] - std::vector<std::string> vStrInputParts; - boost::split(vStrInputParts, strInput, boost::is_any_of(":")); + std::vector<std::string> vStrInputParts = SplitString(strInput, ':'); if (vStrInputParts.size() < 2) throw std::runtime_error("TX output missing separator"); @@ -674,7 +667,7 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) SignatureData sigdata = DataFromTransaction(mergedTx, i, coin.out); // Only sign SIGHASH_SINGLE if there's a corresponding output: if (!fHashSingle || (i < mergedTx.vout.size())) - ProduceSignature(keystore, MutableTransactionSignatureCreator(&mergedTx, i, amount, nHashType), prevPubKey, sigdata); + ProduceSignature(keystore, MutableTransactionSignatureCreator(mergedTx, i, amount, nHashType), prevPubKey, sigdata); if (amount == MAX_MONEY && !sigdata.scriptWitness.IsNull()) { throw std::runtime_error(strprintf("Missing amount for CTxOut with scriptPubKey=%s", HexStr(prevPubKey))); @@ -750,7 +743,7 @@ static void MutateTx(CMutableTransaction& tx, const std::string& command, static void OutputTxJSON(const CTransaction& tx) { UniValue entry(UniValue::VOBJ); - TxToUniv(tx, uint256(), entry); + TxToUniv(tx, /*block_hash=*/uint256(), entry); std::string jsonOutput = entry.write(4); tfm::format(std::cout, "%s\n", jsonOutput); diff --git a/src/bitcoin-util.cpp b/src/bitcoin-util.cpp index b457e0b354..a2f3fca26c 100644 --- a/src/bitcoin-util.cpp +++ b/src/bitcoin-util.cpp @@ -22,8 +22,6 @@ #include <memory> #include <thread> -#include <boost/algorithm/string.hpp> - static const int CONTINUE_EXECUTION=-1; const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 9843382682..bc063faed1 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -20,6 +20,7 @@ #include <util/check.h> #include <util/strencodings.h> #include <util/syscall_sandbox.h> +#include <util/syserror.h> #include <util/system.h> #include <util/threadnames.h> #include <util/tokenpipe.h> @@ -206,7 +207,7 @@ static bool AppInit(NodeContext& node, int argc, char* argv[]) } break; case -1: // Error happened. - return InitError(Untranslated(strprintf("fork_daemon() failed: %s\n", strerror(errno)))); + return InitError(Untranslated(strprintf("fork_daemon() failed: %s\n", SysErrorString(errno)))); default: { // Parent: wait and exit. int token = daemon_ep.TokenRead(); if (token) { // Success diff --git a/src/chainparams.cpp b/src/chainparams.cpp index d3ae6f4cb2..7a7c72ea25 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -9,13 +9,12 @@ #include <consensus/merkle.h> #include <deploymentinfo.h> #include <hash.h> // for signet block challenge hash +#include <script/interpreter.h> +#include <util/string.h> #include <util/system.h> #include <assert.h> -#include <boost/algorithm/string/classification.hpp> -#include <boost/algorithm/string/split.hpp> - static CBlock CreateGenesisBlock(const char* pszTimestamp, const CScript& genesisOutputScript, uint32_t nTime, uint32_t nNonce, uint32_t nBits, int32_t nVersion, const CAmount& genesisReward) { CMutableTransaction txNew; @@ -65,7 +64,10 @@ public: consensus.signet_blocks = false; consensus.signet_challenge.clear(); consensus.nSubsidyHalvingInterval = 210000; - consensus.BIP16Exception = uint256S("0x00000000000002dc756eebf4f49723ed8d30cc28a5f108eb94b1ba88ac4f9c22"); + consensus.script_flag_exceptions.emplace( // BIP16 exception + uint256S("0x00000000000002dc756eebf4f49723ed8d30cc28a5f108eb94b1ba88ac4f9c22"), SCRIPT_VERIFY_NONE); + consensus.script_flag_exceptions.emplace( // Taproot exception + uint256S("0x0000000000000000000f14c35b2d841e986ab5441de8c585d5ffe55ea1e395ad"), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS); consensus.BIP34Height = 227931; consensus.BIP34Hash = uint256S("0x000000000000024b89b42a942fe0d9fea3bb44ab7bd1b19115dd6a759c0808b8"); consensus.BIP65Height = 388381; // 000000000000000004c2b624ed5d7756c508d90fd0da2c7c679febfa6c4735f0 @@ -184,7 +186,8 @@ public: consensus.signet_blocks = false; consensus.signet_challenge.clear(); consensus.nSubsidyHalvingInterval = 210000; - consensus.BIP16Exception = uint256S("0x00000000dd30457c001f4095d208cc1296b0eed002427aa599874af7a432b105"); + consensus.script_flag_exceptions.emplace( // BIP16 exception + uint256S("0x00000000dd30457c001f4095d208cc1296b0eed002427aa599874af7a432b105"), SCRIPT_VERIFY_NONE); consensus.BIP34Height = 21111; consensus.BIP34Hash = uint256S("0x0000000023b3a96d3484e5abb3755c413e7d41500f8e2a5c3f0dd01299cd8ef8"); consensus.BIP65Height = 581885; // 00000000007f6655f22f98e72ed80d8b06dc761d5da09df0fa1dc4be4f861eb6 @@ -323,7 +326,6 @@ public: consensus.signet_blocks = true; consensus.signet_challenge.assign(bin.begin(), bin.end()); consensus.nSubsidyHalvingInterval = 210000; - consensus.BIP16Exception = uint256{}; consensus.BIP34Height = 1; consensus.BIP34Hash = uint256{}; consensus.BIP65Height = 1; @@ -391,13 +393,12 @@ public: consensus.signet_blocks = false; consensus.signet_challenge.clear(); consensus.nSubsidyHalvingInterval = 150; - consensus.BIP16Exception = uint256(); consensus.BIP34Height = 1; // Always active unless overridden consensus.BIP34Hash = uint256(); consensus.BIP65Height = 1; // Always active unless overridden consensus.BIP66Height = 1; // Always active unless overridden consensus.CSVHeight = 1; // Always active unless overridden - consensus.SegwitHeight = 1; // Always active unless overridden + consensus.SegwitHeight = 0; // Always active unless overridden consensus.MinBIP9WarningHeight = 0; consensus.powLimit = uint256S("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); consensus.nPowTargetTimespan = 14 * 24 * 60 * 60; // two weeks @@ -525,8 +526,7 @@ void CRegTestParams::UpdateActivationParametersFromArgs(const ArgsManager& args) if (!args.IsArgSet("-vbparams")) return; for (const std::string& strDeployment : args.GetArgs("-vbparams")) { - std::vector<std::string> vDeploymentParams; - boost::split(vDeploymentParams, strDeployment, boost::is_any_of(":")); + std::vector<std::string> vDeploymentParams = SplitString(strDeployment, ':'); if (vDeploymentParams.size() < 3 || 4 < vDeploymentParams.size()) { throw std::runtime_error("Version bits parameters malformed, expecting deployment:start:end[:min_activation_height]"); } diff --git a/src/compat.h b/src/compat.h index 237b881b11..3ec4ab53fd 100644 --- a/src/compat.h +++ b/src/compat.h @@ -80,10 +80,6 @@ typedef int32_t ssize_t; #endif #endif -#if HAVE_DECL_STRNLEN == 0 -size_t strnlen( const char *start, size_t max_len); -#endif // HAVE_DECL_STRNLEN - #ifndef WIN32 typedef void* sockopt_arg_type; #else diff --git a/src/compat/byteswap.h b/src/compat/byteswap.h index 27ef1a18df..2f4232fa5c 100644 --- a/src/compat/byteswap.h +++ b/src/compat/byteswap.h @@ -9,7 +9,7 @@ #include <config/bitcoin-config.h> #endif -#include <stdint.h> +#include <cstdint> #if defined(HAVE_BYTESWAP_H) #include <byteswap.h> diff --git a/src/compat/cpuid.h b/src/compat/cpuid.h index 0877ad47d3..e78c1ce6d1 100644 --- a/src/compat/cpuid.h +++ b/src/compat/cpuid.h @@ -10,6 +10,8 @@ #include <cpuid.h> +#include <cstdint> + // We can't use cpuid.h's __get_cpuid as it does not support subleafs. void static inline GetCPUID(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) { diff --git a/src/compat/endian.h b/src/compat/endian.h index c5cf7a46cc..bdd8b84c1b 100644 --- a/src/compat/endian.h +++ b/src/compat/endian.h @@ -11,7 +11,7 @@ #include <compat/byteswap.h> -#include <stdint.h> +#include <cstdint> #if defined(HAVE_ENDIAN_H) #include <endian.h> diff --git a/src/compat/glibcxx_sanity.cpp b/src/compat/glibcxx_sanity.cpp index e6e6208e40..f2ceeeeb9c 100644 --- a/src/compat/glibcxx_sanity.cpp +++ b/src/compat/glibcxx_sanity.cpp @@ -5,6 +5,7 @@ #include <list> #include <locale> #include <stdexcept> +#include <string> namespace { diff --git a/src/compat/stdin.cpp b/src/compat/stdin.cpp index 0fc4e0fcf2..61d66e39f2 100644 --- a/src/compat/stdin.cpp +++ b/src/compat/stdin.cpp @@ -2,23 +2,19 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif +#include <compat/stdin.h> -#include <cstdio> // for fileno(), stdin +#include <cstdio> #ifdef WIN32 -#include <windows.h> // for SetStdinEcho() -#include <io.h> // for isatty() +#include <windows.h> +#include <io.h> #else -#include <termios.h> // for SetStdinEcho() -#include <unistd.h> // for SetStdinEcho(), isatty() -#include <poll.h> // for StdinReady() +#include <termios.h> +#include <unistd.h> +#include <poll.h> #endif -#include <compat/stdin.h> - // https://stackoverflow.com/questions/1413445/reading-a-password-from-stdcin void SetStdinEcho(bool enable) { diff --git a/src/compat/strnlen.cpp b/src/compat/strnlen.cpp deleted file mode 100644 index 93a034a664..0000000000 --- a/src/compat/strnlen.cpp +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif - -#include <cstring> - -#if HAVE_DECL_STRNLEN == 0 -size_t strnlen( const char *start, size_t max_len) -{ - const char *end = (const char *)memchr(start, '\0', max_len); - - return end ? (size_t)(end - start) : max_len; -} -#endif // HAVE_DECL_STRNLEN diff --git a/src/consensus/params.h b/src/consensus/params.h index 1ed5ca469f..0881f2e388 100644 --- a/src/consensus/params.h +++ b/src/consensus/params.h @@ -7,7 +7,9 @@ #define BITCOIN_CONSENSUS_PARAMS_H #include <uint256.h> + #include <limits> +#include <map> namespace Consensus { @@ -70,8 +72,13 @@ struct BIP9Deployment { struct Params { uint256 hashGenesisBlock; int nSubsidyHalvingInterval; - /* Block hash that is excepted from BIP16 enforcement */ - uint256 BIP16Exception; + /** + * Hashes of blocks that + * - are known to be consensus valid, and + * - buried in the chain, and + * - fail if the default script verify flags are applied. + */ + std::map<uint256, uint32_t> script_flag_exceptions; /** Block height and hash at which BIP34 becomes active */ int BIP34Height; uint256 BIP34Hash; diff --git a/src/consensus/tx_verify.cpp b/src/consensus/tx_verify.cpp index 5738c333ce..154146f08d 100644 --- a/src/consensus/tx_verify.cpp +++ b/src/consensus/tx_verify.cpp @@ -11,6 +11,7 @@ #include <consensus/validation.h> #include <primitives/transaction.h> #include <script/interpreter.h> +#include <util/check.h> #include <util/moneystr.h> bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime) @@ -74,7 +75,7 @@ std::pair<int, int64_t> CalculateSequenceLocks(const CTransaction &tx, int flags int nCoinHeight = prevHeights[txinIndex]; if (txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG) { - int64_t nCoinTime = block.GetAncestor(std::max(nCoinHeight-1, 0))->GetMedianTimePast(); + const int64_t nCoinTime{Assert(block.GetAncestor(std::max(nCoinHeight - 1, 0)))->GetMedianTimePast()}; // NOTE: Subtract 1 to maintain nLockTime semantics // BIP 68 relative lock times have the semantics of calculating // the first block or time at which the transaction would be diff --git a/src/core_io.h b/src/core_io.h index 69b9ac3ebd..aa1381c374 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -53,8 +53,7 @@ UniValue ValueFromAmount(const CAmount amount); std::string FormatScript(const CScript& script); std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags = 0); std::string SighashToStr(unsigned char sighash_type); -void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool include_hex, bool include_address = true); -void ScriptToUniv(const CScript& script, UniValue& out); -void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex = true, int serialize_flags = 0, const CTxUndo* txundo = nullptr, TxVerbosity verbosity = TxVerbosity::SHOW_DETAILS); +void ScriptToUniv(const CScript& script, UniValue& out, bool include_hex = true, bool include_address = false); +void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex = true, int serialize_flags = 0, const CTxUndo* txundo = nullptr, TxVerbosity verbosity = TxVerbosity::SHOW_DETAILS); #endif // BITCOIN_CORE_IO_H diff --git a/src/core_read.cpp b/src/core_read.cpp index 3bab5b5d98..77c516427a 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -14,9 +14,6 @@ #include <util/strencodings.h> #include <version.h> -#include <boost/algorithm/string/classification.hpp> -#include <boost/algorithm/string/split.hpp> - #include <algorithm> #include <string> @@ -66,12 +63,11 @@ CScript ParseScript(const std::string& s) { CScript result; - std::vector<std::string> words; - boost::algorithm::split(words, s, boost::algorithm::is_any_of(" \t\n"), boost::algorithm::token_compress_on); + std::vector<std::string> words = SplitString(s, " \t\n"); for (const std::string& w : words) { if (w.empty()) { - // Empty string, ignore. (boost::split given '' will return one word) + // Empty string, ignore. (SplitString doesn't combine multiple separators) } else if (std::all_of(w.begin(), w.end(), ::IsDigit) || (w.front() == '-' && w.size() > 1 && std::all_of(w.begin() + 1, w.end(), ::IsDigit))) { diff --git a/src/core_write.cpp b/src/core_write.cpp index c4b6b8d27e..cfd4cb1b49 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -19,6 +19,10 @@ #include <util/strencodings.h> #include <util/system.h> +#include <map> +#include <string> +#include <vector> + UniValue ValueFromAmount(const CAmount amount) { static_assert(COIN > 1); @@ -143,29 +147,28 @@ std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags) return HexStr(ssTx); } -void ScriptToUniv(const CScript& script, UniValue& out) -{ - ScriptPubKeyToUniv(script, out, /* include_hex */ true, /* include_address */ false); -} - -void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool include_hex, bool include_address) +void ScriptToUniv(const CScript& script, UniValue& out, bool include_hex, bool include_address) { CTxDestination address; - out.pushKV("asm", ScriptToAsmStr(scriptPubKey)); - out.pushKV("desc", InferDescriptor(scriptPubKey, DUMMY_SIGNING_PROVIDER)->ToString()); - if (include_hex) out.pushKV("hex", HexStr(scriptPubKey)); + out.pushKV("asm", ScriptToAsmStr(script)); + if (include_address) { + out.pushKV("desc", InferDescriptor(script, DUMMY_SIGNING_PROVIDER)->ToString()); + } + if (include_hex) { + out.pushKV("hex", HexStr(script)); + } std::vector<std::vector<unsigned char>> solns; - const TxoutType type{Solver(scriptPubKey, solns)}; + const TxoutType type{Solver(script, solns)}; - if (include_address && ExtractDestination(scriptPubKey, address) && type != TxoutType::PUBKEY) { + if (include_address && ExtractDestination(script, address) && type != TxoutType::PUBKEY) { out.pushKV("address", EncodeDestination(address)); } out.pushKV("type", GetTxnOutputType(type)); } -void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex, int serialize_flags, const CTxUndo* txundo, TxVerbosity verbosity) +void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex, int serialize_flags, const CTxUndo* txundo, TxVerbosity verbosity) { entry.pushKV("txid", tx.GetHash().GetHex()); entry.pushKV("hash", tx.GetWitnessHash().GetHex()); @@ -213,7 +216,7 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, if (verbosity == TxVerbosity::SHOW_DETAILS_AND_PREVOUT) { UniValue o_script_pub_key(UniValue::VOBJ); - ScriptPubKeyToUniv(prev_txout.scriptPubKey, o_script_pub_key, /*include_hex=*/ true); + ScriptToUniv(prev_txout.scriptPubKey, /*out=*/o_script_pub_key, /*include_hex=*/true, /*include_address=*/true); UniValue p(UniValue::VOBJ); p.pushKV("generated", bool(prev_coin.fCoinBase)); @@ -238,7 +241,7 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, out.pushKV("n", (int64_t)i); UniValue o(UniValue::VOBJ); - ScriptPubKeyToUniv(txout.scriptPubKey, o, true); + ScriptToUniv(txout.scriptPubKey, /*out=*/o, /*include_hex=*/true, /*include_address=*/true); out.pushKV("scriptPubKey", o); vout.push_back(out); @@ -254,8 +257,9 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, entry.pushKV("fee", ValueFromAmount(fee)); } - if (!hashBlock.IsNull()) - entry.pushKV("blockhash", hashBlock.GetHex()); + if (!block_hash.IsNull()) { + entry.pushKV("blockhash", block_hash.GetHex()); + } if (include_hex) { entry.pushKV("hex", EncodeHexTx(tx, serialize_flags)); // The hex-encoded transaction. Used the name "hex" to be consistent with the verbose output of "getrawtransaction". diff --git a/src/crypto/chacha20.cpp b/src/crypto/chacha20.cpp index f3ff4268ee..c7e12b0612 100644 --- a/src/crypto/chacha20.cpp +++ b/src/crypto/chacha20.cpp @@ -18,6 +18,8 @@ constexpr static inline uint32_t rotl32(uint32_t v, int c) { return (v << c) | ( a += b; d = rotl32(d ^ a, 8); \ c += d; b = rotl32(b ^ c, 7); +#define REPEAT10(a) do { {a}; {a}; {a}; {a}; {a}; {a}; {a}; {a}; {a}; {a}; } while(0) + static const unsigned char sigma[] = "expand 32-byte k"; static const unsigned char tau[] = "expand 16-byte k"; @@ -119,16 +121,19 @@ void ChaCha20::Keystream(unsigned char* c, size_t bytes) x13 = j13; x14 = j14; x15 = j15; - for (i = 20;i > 0;i -= 2) { - QUARTERROUND( x0, x4, x8,x12) - QUARTERROUND( x1, x5, x9,x13) - QUARTERROUND( x2, x6,x10,x14) - QUARTERROUND( x3, x7,x11,x15) - QUARTERROUND( x0, x5,x10,x15) - QUARTERROUND( x1, x6,x11,x12) - QUARTERROUND( x2, x7, x8,x13) - QUARTERROUND( x3, x4, x9,x14) - } + + // The 20 inner ChaCha20 rounds are unrolled here for performance. + REPEAT10( + QUARTERROUND( x0, x4, x8,x12); + QUARTERROUND( x1, x5, x9,x13); + QUARTERROUND( x2, x6,x10,x14); + QUARTERROUND( x3, x7,x11,x15); + QUARTERROUND( x0, x5,x10,x15); + QUARTERROUND( x1, x6,x11,x12); + QUARTERROUND( x2, x7, x8,x13); + QUARTERROUND( x3, x4, x9,x14); + ); + x0 += j0; x1 += j1; x2 += j2; @@ -231,16 +236,19 @@ void ChaCha20::Crypt(const unsigned char* m, unsigned char* c, size_t bytes) x13 = j13; x14 = j14; x15 = j15; - for (i = 20;i > 0;i -= 2) { - QUARTERROUND( x0, x4, x8,x12) - QUARTERROUND( x1, x5, x9,x13) - QUARTERROUND( x2, x6,x10,x14) - QUARTERROUND( x3, x7,x11,x15) - QUARTERROUND( x0, x5,x10,x15) - QUARTERROUND( x1, x6,x11,x12) - QUARTERROUND( x2, x7, x8,x13) - QUARTERROUND( x3, x4, x9,x14) - } + + // The 20 inner ChaCha20 rounds are unrolled here for performance. + REPEAT10( + QUARTERROUND( x0, x4, x8,x12); + QUARTERROUND( x1, x5, x9,x13); + QUARTERROUND( x2, x6,x10,x14); + QUARTERROUND( x3, x7,x11,x15); + QUARTERROUND( x0, x5,x10,x15); + QUARTERROUND( x1, x6,x11,x12); + QUARTERROUND( x2, x7, x8,x13); + QUARTERROUND( x3, x4, x9,x14); + ); + x0 += j0; x1 += j1; x2 += j2; diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index cde543e68c..196f81ea16 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -586,17 +586,9 @@ std::string SHA256AutoDetect() bool have_sse4 = false; bool have_xsave = false; bool have_avx = false; - bool have_avx2 = false; - bool have_x86_shani = false; - bool enabled_avx = false; - - (void)AVXEnabled; - (void)have_sse4; - (void)have_avx; - (void)have_xsave; - (void)have_avx2; - (void)have_x86_shani; - (void)enabled_avx; + [[maybe_unused]] bool have_avx2 = false; + [[maybe_unused]] bool have_x86_shani = false; + [[maybe_unused]] bool enabled_avx = false; uint32_t eax, ebx, ecx, edx; GetCPUID(1, 0, eax, ebx, ecx, edx); @@ -641,7 +633,7 @@ std::string SHA256AutoDetect() ret += ",avx2(8way)"; } #endif -#endif +#endif // defined(USE_ASM) && defined(HAVE_GETCPUID) #if defined(ENABLE_ARM_SHANI) && !defined(BUILD_BITCOIN_INTERNAL) bool have_arm_shani = false; diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index b0ea80ea1a..50a601c684 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -227,7 +227,7 @@ const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8; std::vector<unsigned char> CDBWrapper::CreateObfuscateKey() const { std::vector<uint8_t> ret(OBFUSCATE_KEY_NUM_BYTES); - GetRandBytes(ret.data(), OBFUSCATE_KEY_NUM_BYTES); + GetRandBytes(ret); return ret; } diff --git a/src/dummywallet.cpp b/src/dummywallet.cpp index 2b94ed611b..028c6ebae1 100644 --- a/src/dummywallet.cpp +++ b/src/dummywallet.cpp @@ -50,6 +50,7 @@ void DummyWalletInit::AddWalletOptions(ArgsManager& argsman) const "-flushwallet", "-privdb", "-walletrejectlongchains", + "-walletcrosschain", "-unsafesqlitesync", }); } diff --git a/src/flatfile.cpp b/src/flatfile.cpp index d6cada0c46..0fecf4f504 100644 --- a/src/flatfile.cpp +++ b/src/flatfile.cpp @@ -27,7 +27,7 @@ std::string FlatFilePos::ToString() const fs::path FlatFileSeq::FileName(const FlatFilePos& pos) const { - return m_dir / strprintf("%s%05u.dat", m_prefix, pos.nFile); + return m_dir / fs::u8path(strprintf("%s%05u.dat", m_prefix, pos.nFile)); } FILE* FlatFileSeq::Open(const FlatFilePos& pos, bool read_only) diff --git a/src/fs.cpp b/src/fs.cpp index 219fdee959..b61115bf01 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <fs.h> +#include <util/syserror.h> #ifndef WIN32 #include <cstring> @@ -44,7 +45,7 @@ fs::path AbsPathJoin(const fs::path& base, const fs::path& path) static std::string GetErrorReason() { - return std::strerror(errno); + return SysErrorString(errno); } FileLock::FileLock(const fs::path& file) @@ -51,12 +51,26 @@ public: // Disallow std::string conversion method to avoid locale-dependent encoding on windows. std::string string() const = delete; + std::string u8string() const + { + const auto& utf8_str{std::filesystem::path::u8string()}; + // utf8_str might either be std::string (C++17) or std::u8string + // (C++20). Convert both to std::string. This method can be removed + // after switching to C++20. + return std::string{utf8_str.begin(), utf8_str.end()}; + } + // Required for path overloads in <fstream>. // See https://gcc.gnu.org/git/?p=gcc.git;a=commit;h=96e0367ead5d8dcac3bec2865582e76e2fbab190 path& make_preferred() { std::filesystem::path::make_preferred(); return *this; } path filename() const { return std::filesystem::path::filename(); } }; +static inline path u8path(const std::string& utf8_str) +{ + return std::filesystem::u8path(utf8_str); +} + // Disallow implicit std::string conversion for absolute to avoid // locale-dependent encoding on windows. static inline path absolute(const path& p) @@ -78,11 +92,30 @@ static inline auto quoted(const std::string& s) } // Allow safe path append operations. -static inline path operator+(path p1, path p2) +static inline path operator/(path p1, path p2) +{ + p1 /= std::move(p2); + return p1; +} +static inline path operator/(path p1, const char* p2) +{ + p1 /= p2; + return p1; +} +static inline path operator+(path p1, const char* p2) { - p1 += std::move(p2); + p1 += p2; return p1; } +static inline path operator+(path p1, path::value_type p2) +{ + p1 += p2; + return p1; +} + +// Disallow unsafe path append operations. +template<typename T> static inline path operator/(path p1, T p2) = delete; +template<typename T> static inline path operator+(path p1, T p2) = delete; // Disallow implicit std::string conversion for copy_file // to avoid locale-dependent encoding on Windows. @@ -116,8 +149,8 @@ static inline std::string PathToString(const path& path) // use here, because these methods encode the path using C++'s narrow // multibyte encoding, which on Windows corresponds to the current "code // page", which is unpredictable and typically not able to represent all - // valid paths. So std::filesystem::path::u8string() and - // std::filesystem::u8path() functions are used instead on Windows. On + // valid paths. So fs::path::u8string() and + // fs::u8path() functions are used instead on Windows. On // POSIX, u8string/u8path functions are not safe to use because paths are // not always valid UTF-8, so plain string methods which do not transform // the path there are used. diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 5d0b59f7cb..3bf165495c 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -4,7 +4,6 @@ #include <httprpc.h> -#include <chainparams.h> #include <crypto/hmac_sha256.h> #include <httpserver.h> #include <rpc/protocol.h> @@ -12,18 +11,15 @@ #include <util/strencodings.h> #include <util/string.h> #include <util/system.h> -#include <util/translation.h> #include <walletinitinterface.h> #include <algorithm> #include <iterator> #include <map> #include <memory> -#include <stdio.h> #include <set> #include <string> - -#include <boost/algorithm/string.hpp> +#include <vector> /** WWW-Authenticate to present with 401 Unauthorized response */ static const char* WWW_AUTH_HEADER_DATA = "Basic realm=\"jsonrpc\""; @@ -131,8 +127,11 @@ static bool RPCAuthorized(const std::string& strAuth, std::string& strAuthUserna return false; if (strAuth.substr(0, 6) != "Basic ") return false; - std::string strUserPass64 = TrimString(strAuth.substr(6)); - std::string strUserPass = DecodeBase64(strUserPass64); + std::string_view strUserPass64 = TrimStringView(std::string_view{strAuth}.substr(6)); + auto userpass_data = DecodeBase64(strUserPass64); + std::string strUserPass; + if (!userpass_data) return false; + strUserPass.assign(userpass_data->begin(), userpass_data->end()); if (strUserPass.find(':') != std::string::npos) strAuthUsernameOut = strUserPass.substr(0, strUserPass.find(':')); @@ -251,13 +250,14 @@ static bool InitRPCAuthentication() LogPrintf("Config options rpcuser and rpcpassword will soon be deprecated. Locally-run instances may remove rpcuser to use cookie-based auth, or may be replaced with rpcauth. Please see share/rpcauth for rpcauth auth generation.\n"); strRPCUserColonPass = gArgs.GetArg("-rpcuser", "") + ":" + gArgs.GetArg("-rpcpassword", ""); } - if (gArgs.GetArg("-rpcauth","") != "") - { + if (gArgs.GetArg("-rpcauth", "") != "") { LogPrintf("Using rpcauth authentication.\n"); for (const std::string& rpcauth : gArgs.GetArgs("-rpcauth")) { - std::vector<std::string> fields; - boost::split(fields, rpcauth, boost::is_any_of(":$")); - if (fields.size() == 3) { + std::vector<std::string> fields{SplitString(rpcauth, ':')}; + const std::vector<std::string> salt_hmac{SplitString(fields.back(), '$')}; + if (fields.size() == 2 && salt_hmac.size() == 2) { + fields.pop_back(); + fields.insert(fields.end(), salt_hmac.begin(), salt_hmac.end()); g_rpcauth.push_back(fields); } else { LogPrintf("Invalid -rpcauth argument.\n"); @@ -274,8 +274,10 @@ static bool InitRPCAuthentication() std::set<std::string>& whitelist = g_rpc_whitelist[strUser]; if (pos != std::string::npos) { std::string strWhitelist = strRPCWhitelist.substr(pos + 1); - std::set<std::string> new_whitelist; - boost::split(new_whitelist, strWhitelist, boost::is_any_of(", ")); + std::vector<std::string> whitelist_split = SplitString(strWhitelist, ", "); + std::set<std::string> new_whitelist{ + std::make_move_iterator(whitelist_split.begin()), + std::make_move_iterator(whitelist_split.end())}; if (intersect) { std::set<std::string> tmp_whitelist; std::set_intersection(new_whitelist.begin(), new_whitelist.end(), diff --git a/src/httpserver.cpp b/src/httpserver.cpp index e00c68585e..96bee8640d 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -23,6 +23,7 @@ #include <deque> #include <memory> +#include <optional> #include <stdio.h> #include <stdlib.h> #include <string> @@ -30,11 +31,12 @@ #include <sys/types.h> #include <sys/stat.h> -#include <event2/thread.h> #include <event2/buffer.h> #include <event2/bufferevent.h> -#include <event2/util.h> +#include <event2/http.h> #include <event2/keyvalq_struct.h> +#include <event2/thread.h> +#include <event2/util.h> #include <support/events.h> @@ -358,12 +360,8 @@ bool InitHTTPServer() // Redirect libevent's logging to our own log event_set_log_callback(&libevent_log_cb); - // Update libevent's log handling. Returns false if our version of - // libevent doesn't support debug logging, in which case we should - // clear the BCLog::LIBEVENT flag. - if (!UpdateHTTPServerLogging(LogInstance().WillLogCategory(BCLog::LIBEVENT))) { - LogInstance().DisableCategory(BCLog::LIBEVENT); - } + // Update libevent's log handling. + UpdateHTTPServerLogging(LogInstance().WillLogCategory(BCLog::LIBEVENT)); #ifdef WIN32 evthread_use_windows_threads(); @@ -402,18 +400,12 @@ bool InitHTTPServer() return true; } -bool UpdateHTTPServerLogging(bool enable) { -#if LIBEVENT_VERSION_NUMBER >= 0x02010100 +void UpdateHTTPServerLogging(bool enable) { if (enable) { event_enable_debug_logging(EVENT_DBG_ALL); } else { event_enable_debug_logging(EVENT_DBG_NONE); } - return true; -#else - // Can't update libevent logging if version < 02010100 - return false; -#endif } static std::thread g_thread_http; @@ -639,6 +631,37 @@ HTTPRequest::RequestMethod HTTPRequest::GetRequestMethod() const } } +std::optional<std::string> HTTPRequest::GetQueryParameter(const std::string& key) const +{ + const char* uri{evhttp_request_get_uri(req)}; + + return GetQueryParameterFromUri(uri, key); +} + +std::optional<std::string> GetQueryParameterFromUri(const char* uri, const std::string& key) +{ + evhttp_uri* uri_parsed{evhttp_uri_parse(uri)}; + const char* query{evhttp_uri_get_query(uri_parsed)}; + std::optional<std::string> result; + + if (query) { + // Parse the query string into a key-value queue and iterate over it + struct evkeyvalq params_q; + evhttp_parse_query_str(query, ¶ms_q); + + for (struct evkeyval* param{params_q.tqh_first}; param != nullptr; param = param->next.tqe_next) { + if (param->key == key) { + result = param->value; + break; + } + } + evhttp_clear_headers(¶ms_q); + } + evhttp_uri_free(uri_parsed); + + return result; +} + void RegisterHTTPHandler(const std::string &prefix, bool exactMatch, const HTTPRequestHandler &handler) { LogPrint(BCLog::HTTP, "Registering HTTP handler for %s (exactmatch %d)\n", prefix, exactMatch); diff --git a/src/httpserver.h b/src/httpserver.h index 97cd63778a..5ab3f18927 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -5,8 +5,9 @@ #ifndef BITCOIN_HTTPSERVER_H #define BITCOIN_HTTPSERVER_H -#include <string> #include <functional> +#include <optional> +#include <string> static const int DEFAULT_HTTP_THREADS=4; static const int DEFAULT_HTTP_WORKQUEUE=16; @@ -31,9 +32,8 @@ void InterruptHTTPServer(); /** Stop HTTP server */ void StopHTTPServer(); -/** Change logging level for libevent. Removes BCLog::LIBEVENT from log categories if - * libevent doesn't support debug logging.*/ -bool UpdateHTTPServerLogging(bool enable); +/** Change logging level for libevent. */ +void UpdateHTTPServerLogging(bool enable); /** Handler for requests to a certain HTTP path */ typedef std::function<bool(HTTPRequest* req, const std::string &)> HTTPRequestHandler; @@ -83,6 +83,17 @@ public: */ RequestMethod GetRequestMethod() const; + /** Get the query parameter value from request uri for a specified key, or std::nullopt if the + * key is not found. + * + * If the query string contains duplicate keys, the first value is returned. Many web frameworks + * would instead parse this as an array of values, but this is not (yet) implemented as it is + * currently not needed in any of the endpoints. + * + * @param[in] key represents the query parameter of which the value is returned + */ + std::optional<std::string> GetQueryParameter(const std::string& key) const; + /** * Get the request header specified by hdr, or an empty string. * Return a pair (isPresent,string). @@ -115,6 +126,20 @@ public: void WriteReply(int nStatus, const std::string& strReply = ""); }; +/** Get the query parameter value from request uri for a specified key, or std::nullopt if the key + * is not found. + * + * If the query string contains duplicate keys, the first value is returned. Many web frameworks + * would instead parse this as an array of values, but this is not (yet) implemented as it is + * currently not needed in any of the endpoints. + * + * Helper function for HTTPRequest::GetQueryParameter. + * + * @param[in] uri is the entire request uri + * @param[in] key represents the query parameter of which the value is returned + */ +std::optional<std::string> GetQueryParameterFromUri(const char* uri, const std::string& key); + /** Event handler closure. */ class HTTPClosure diff --git a/src/i2p.cpp b/src/i2p.cpp index ccba14d63d..e08b5461fe 100644 --- a/src/i2p.cpp +++ b/src/i2p.cpp @@ -69,12 +69,11 @@ static std::string SwapBase64(const std::string& from) static Binary DecodeI2PBase64(const std::string& i2p_b64) { const std::string& std_b64 = SwapBase64(i2p_b64); - bool invalid; - Binary decoded = DecodeBase64(std_b64.c_str(), &invalid); - if (invalid) { + auto decoded = DecodeBase64(std_b64); + if (!decoded) { throw std::runtime_error(strprintf("Cannot decode Base64: \"%s\"", i2p_b64)); } - return decoded; + return std::move(*decoded); } /** diff --git a/src/index/base.cpp b/src/index/base.cpp index 8fe30f8960..a00ae13e5c 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -18,8 +18,8 @@ using node::ReadBlockFromDisk; constexpr uint8_t DB_BEST_BLOCK{'B'}; -constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds -constexpr int64_t SYNC_LOCATOR_WRITE_INTERVAL = 30; // seconds +constexpr auto SYNC_LOG_INTERVAL{30s}; +constexpr auto SYNC_LOCATOR_WRITE_INTERVAL{30s}; template <typename... Args> static void FatalError(const char* fmt, const Args&... args) @@ -65,9 +65,9 @@ bool BaseIndex::Init() LOCK(cs_main); CChain& active_chain = m_chainstate->m_chain; if (locator.IsNull()) { - m_best_block_index = nullptr; + SetBestBlockIndex(nullptr); } else { - m_best_block_index = m_chainstate->FindForkInGlobalIndex(locator); + SetBestBlockIndex(m_chainstate->FindForkInGlobalIndex(locator)); } m_synced = m_best_block_index.load() == active_chain.Tip(); if (!m_synced) { @@ -75,11 +75,7 @@ bool BaseIndex::Init() if (!m_best_block_index) { // index is not built yet // make sure we have all block data back to the genesis - const CBlockIndex* block = active_chain.Tip(); - while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) { - block = block->pprev; - } - prune_violation = block != active_chain.Genesis(); + prune_violation = m_chainstate->m_blockman.GetFirstStoredBlock(*active_chain.Tip()) != active_chain.Genesis(); } // in case the index has a best block set and is not fully synced // check if we have the required blocks to continue building the index @@ -134,11 +130,11 @@ void BaseIndex::ThreadSync() if (!m_synced) { auto& consensus_params = Params().GetConsensus(); - int64_t last_log_time = 0; - int64_t last_locator_write_time = 0; + std::chrono::steady_clock::time_point last_log_time{0s}; + std::chrono::steady_clock::time_point last_locator_write_time{0s}; while (true) { if (m_interrupt) { - m_best_block_index = pindex; + SetBestBlockIndex(pindex); // No need to handle errors in Commit. If it fails, the error will be already be // logged. The best way to recover is to continue, as index cannot be corrupted by // a missed commit to disk for an advanced index state. @@ -150,7 +146,7 @@ void BaseIndex::ThreadSync() LOCK(cs_main); const CBlockIndex* pindex_next = NextSyncBlock(pindex, m_chainstate->m_chain); if (!pindex_next) { - m_best_block_index = pindex; + SetBestBlockIndex(pindex); m_synced = true; // No need to handle errors in Commit. See rationale above. Commit(); @@ -164,7 +160,7 @@ void BaseIndex::ThreadSync() pindex = pindex_next; } - int64_t current_time = GetTime(); + auto current_time{std::chrono::steady_clock::now()}; if (last_log_time + SYNC_LOG_INTERVAL < current_time) { LogPrintf("Syncing %s with block chain from height %d\n", GetName(), pindex->nHeight); @@ -172,7 +168,7 @@ void BaseIndex::ThreadSync() } if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) { - m_best_block_index = pindex; + SetBestBlockIndex(pindex); last_locator_write_time = current_time; // No need to handle errors in Commit. See rationale above. Commit(); @@ -230,10 +226,10 @@ bool BaseIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_ti // out of sync may be possible but a users fault. // In case we reorg beyond the pruned depth, ReadBlockFromDisk would // throw and lead to a graceful shutdown - m_best_block_index = new_tip; + SetBestBlockIndex(new_tip); if (!Commit()) { // If commit fails, revert the best block index to avoid corruption. - m_best_block_index = current_tip; + SetBestBlockIndex(current_tip); return false; } @@ -274,7 +270,7 @@ void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const } if (WriteBlock(*block, pindex)) { - m_best_block_index = pindex; + SetBestBlockIndex(pindex); } else { FatalError("%s: Failed to write block %s to index", __func__, pindex->GetBlockHash().ToString()); @@ -381,3 +377,14 @@ IndexSummary BaseIndex::GetSummary() const summary.best_block_height = m_best_block_index ? m_best_block_index.load()->nHeight : 0; return summary; } + +void BaseIndex::SetBestBlockIndex(const CBlockIndex* block) { + assert(!node::fPruneMode || AllowPrune()); + + m_best_block_index = block; + if (AllowPrune() && block) { + node::PruneLockInfo prune_lock; + prune_lock.height_first = block->nHeight; + WITH_LOCK(::cs_main, m_chainstate->m_blockman.UpdatePruneLock(GetName(), prune_lock)); + } +} diff --git a/src/index/base.h b/src/index/base.h index c4a8215bc4..a8f6a18c8d 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -75,6 +75,9 @@ private: /// to a chain reorganization), the index must halt until Commit succeeds or else it could end up /// getting corrupted. bool Commit(); + + virtual bool AllowPrune() const = 0; + protected: CChainState* m_chainstate{nullptr}; @@ -103,6 +106,9 @@ protected: /// Get the name of the index for display in logs. virtual const char* GetName() const = 0; + /// Update the internal best block index as well as the prune lock. + void SetBestBlockIndex(const CBlockIndex* block); + public: /// Destructor interrupts sync thread if running and blocks until it exits. virtual ~BaseIndex(); diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index 4f99eddfd7..c92b8c7e19 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -100,7 +100,7 @@ BlockFilterIndex::BlockFilterIndex(BlockFilterType filter_type, const std::string& filter_name = BlockFilterTypeName(filter_type); if (filter_name.empty()) throw std::invalid_argument("unknown filter_type"); - fs::path path = gArgs.GetDataDirNet() / "indexes" / "blockfilter" / filter_name; + fs::path path = gArgs.GetDataDirNet() / "indexes" / "blockfilter" / fs::u8path(filter_name); fs::create_directories(path); m_name = filter_name + " block filter index"; diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h index a049019c02..b1836fe12f 100644 --- a/src/index/blockfilterindex.h +++ b/src/index/blockfilterindex.h @@ -38,6 +38,8 @@ private: /** cache of block hash to filter header, to avoid disk access when responding to getcfcheckpt. */ std::unordered_map<uint256, uint256, FilterHeaderHasher> m_headers_cache GUARDED_BY(m_cs_headers_cache); + bool AllowPrune() const override { return true; } + protected: bool Init() override; diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index a1c8a5937c..69078708f9 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -228,10 +228,9 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) m_muhash.Finalize(out); value.second.muhash = out; - CDBBatch batch(*m_db); - batch.Write(DBHeightKey(pindex->nHeight), value); - batch.Write(DB_MUHASH, m_muhash); - return m_db->WriteBatch(batch); + // Intentionally do not update DB_MUHASH here so it stays in sync with + // DB_BEST_BLOCK, and the index is not corrupted if there is an unclean shutdown. + return m_db->Write(DBHeightKey(pindex->nHeight), value); } static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch, @@ -278,7 +277,7 @@ bool CoinStatsIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* n { LOCK(cs_main); - CBlockIndex* iter_tip{m_chainstate->m_blockman.LookupBlockIndex(current_tip->GetBlockHash())}; + const CBlockIndex* iter_tip{m_chainstate->m_blockman.LookupBlockIndex(current_tip->GetBlockHash())}; const auto& consensus_params{Params().GetConsensus()}; do { @@ -388,6 +387,14 @@ bool CoinStatsIndex::Init() return true; } +bool CoinStatsIndex::CommitInternal(CDBBatch& batch) +{ + // DB_MUHASH should always be committed in a batch together with DB_BEST_BLOCK + // to prevent an inconsistent state of the DB. + batch.Write(DB_MUHASH, m_muhash); + return BaseIndex::CommitInternal(batch); +} + // Reverse a single block as part of a reorg bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex) { @@ -489,5 +496,5 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex Assert(m_total_unspendables_scripts == read_out.second.total_unspendables_scripts); Assert(m_total_unspendables_unclaimed_rewards == read_out.second.total_unspendables_unclaimed_rewards); - return m_db->Write(DB_MUHASH, m_muhash); + return true; } diff --git a/src/index/coinstatsindex.h b/src/index/coinstatsindex.h index d2a6c9c964..6f53bb74fb 100644 --- a/src/index/coinstatsindex.h +++ b/src/index/coinstatsindex.h @@ -36,9 +36,13 @@ private: bool ReverseBlock(const CBlock& block, const CBlockIndex* pindex); + bool AllowPrune() const override { return true; } + protected: bool Init() override; + bool CommitInternal(CDBBatch& batch) override; + bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) override; diff --git a/src/index/txindex.h b/src/index/txindex.h index 2bbc602631..ec339abaa1 100644 --- a/src/index/txindex.h +++ b/src/index/txindex.h @@ -20,6 +20,8 @@ protected: private: const std::unique_ptr<DB> m_db; + bool AllowPrune() const override { return false; } + protected: bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; diff --git a/src/init.cpp b/src/init.cpp index a3d53c3fae..713598f411 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -33,6 +33,7 @@ #include <net_permissions.h> #include <net_processing.h> #include <netbase.h> +#include <netgroup.h> #include <node/blockstorage.h> #include <node/caches.h> #include <node/chainstate.h> @@ -64,6 +65,7 @@ #include <util/strencodings.h> #include <util/string.h> #include <util/syscall_sandbox.h> +#include <util/syserror.h> #include <util/system.h> #include <util/thread.h> #include <util/threadnames.h> @@ -72,6 +74,7 @@ #include <validationinterface.h> #include <walletinitinterface.h> +#include <algorithm> #include <condition_variable> #include <cstdint> #include <cstdio> @@ -89,7 +92,6 @@ #include <sys/stat.h> #endif -#include <boost/algorithm/string/replace.hpp> #include <boost/signals2/signal.hpp> #if ENABLE_ZMQ @@ -109,7 +111,6 @@ using node::LoadChainstate; using node::NodeContext; using node::ThreadImport; using node::VerifyLoadedChainstate; -using node::fHavePruned; using node::fPruneMode; using node::fReindex; using node::nPruneTarget; @@ -135,7 +136,7 @@ static const char* BITCOIN_PID_FILENAME = "bitcoind.pid"; static fs::path GetPidFile(const ArgsManager& args) { - return AbsPathForConfigVal(fs::PathFromString(args.GetArg("-pid", BITCOIN_PID_FILENAME))); + return AbsPathForConfigVal(args.GetPathArg("-pid", BITCOIN_PID_FILENAME)); } [[nodiscard]] static bool CreatePidFile(const ArgsManager& args) @@ -149,7 +150,7 @@ static fs::path GetPidFile(const ArgsManager& args) #endif return true; } else { - return InitError(strprintf(_("Unable to create the PID file '%s': %s"), fs::PathToString(GetPidFile(args)), std::strerror(errno))); + return InitError(strprintf(_("Unable to create the PID file '%s': %s"), fs::PathToString(GetPidFile(args)), SysErrorString(errno))); } } @@ -240,6 +241,7 @@ void Shutdown(NodeContext& node) node.connman.reset(); node.banman.reset(); node.addrman.reset(); + node.netgroupman.reset(); if (node.mempool && node.mempool->IsLoaded() && node.args->GetBoolArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { DumpMempool(*node.mempool); @@ -406,7 +408,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-blockreconstructionextratxn=<n>", strprintf("Extra transactions to keep in memory for compact block reconstructions (default: %u)", DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-blocksonly", strprintf("Whether to reject transactions from network peers. Automatic broadcast and rebroadcast of any transactions from inbound peers is disabled, unless the peer has the 'forcerelay' permission. RPC transactions are not affected. (default: %u)", DEFAULT_BLOCKSONLY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-coinstatsindex", strprintf("Maintain coinstats index used by the gettxoutsetinfo RPC (default: %u)", DEFAULT_COINSTATSINDEX), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-conf=<file>", strprintf("Specify path to read-only configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-conf=<file>", strprintf("Specify path to read-only configuration file. Relative paths will be prefixed by datadir location (only useable from command line, not configuration file) (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); argsman.AddArg("-dbcache=<n>", strprintf("Maximum database cache size <n> MiB (%d to %d, default: %d). In addition, unused mempool memory is shared for this cache (see -maxmempool).", nMinDbCache, nMaxDbCache, nDefaultDbCache), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -420,11 +422,11 @@ void SetupServerArgs(ArgsManager& argsman) -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-persistmempool", strprintf("Whether to save the mempool on shutdown and load on restart (default: %u)", DEFAULT_PERSIST_MEMPOOL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-pid=<file>", strprintf("Specify pid file. Relative paths will be prefixed by a net-specific datadir location. (default: %s)", BITCOIN_PID_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex and -coinstatsindex. " + argsman.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex. " "Warning: Reverting this setting requires re-downloading the entire blockchain. " "(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >=%u = automatically prune block files to stay under the specified target size in MiB)", MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-reindex", "Rebuild chain state and block index from the blk*.dat files on disk", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-reindex-chainstate", "Rebuild chain state from the currently indexed blocks. When in pruning mode or if blocks on disk might be corrupted, use full -reindex instead.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-reindex", "Rebuild chain state and block index from the blk*.dat files on disk. This will also rebuild active optional indexes.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-reindex-chainstate", "Rebuild chain state from the currently indexed blocks. When in pruning mode or if blocks on disk might be corrupted, use full -reindex instead. Deactivate all optional indexes before running this.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-settings=<file>", strprintf("Specify path to dynamic settings data file. Can be disabled with -nosettings. File is written at runtime and not meant to be edited by users (use %s instead for custom settings). Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME, BITCOIN_SETTINGS_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); #if HAVE_SYSTEM argsman.AddArg("-startupnotify=<cmd>", "Execute command on startup.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -444,7 +446,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-asmap=<file>", strprintf("Specify asn mapping used for bucketing of the peers (default: %s). Relative paths will be prefixed by the net-specific datadir location.", DEFAULT_ASMAP_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-bantime=<n>", strprintf("Default duration (in seconds) of manually configured bans (default: %u)", DEFAULT_MISBEHAVING_BANTIME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-bind=<addr>[:<port>][=onion]", strprintf("Bind to given address and always listen on it (default: 0.0.0.0). Use [host]:port notation for IPv6. Append =onion to tag any incoming connections to that address and port as incoming Tor connections (default: 127.0.0.1:%u=onion, testnet: 127.0.0.1:%u=onion, signet: 127.0.0.1:%u=onion, regtest: 127.0.0.1:%u=onion)", defaultBaseParams->OnionServiceTargetPort(), testnetBaseParams->OnionServiceTargetPort(), signetBaseParams->OnionServiceTargetPort(), regtestBaseParams->OnionServiceTargetPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); - argsman.AddArg("-cjdnsreachable", "If set then this host is configured for CJDNS (connecting to fc00::/8 addresses would lead us to the CJDNS network) (default: 0)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-cjdnsreachable", "If set, then this host is configured for CJDNS (connecting to fc00::/8 addresses would lead us to the CJDNS network, see doc/cjdns.md) (default: 0)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-connect=<ip>", "Connect only to the specified node; -noconnect disables automatic connections (the rules for this peer are the same as for -addnode). This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-discover", "Discover own IP addresses (default: 1 when listening and no -externalip or -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-dns", strprintf("Allow DNS lookups for -addnode, -seednode and -connect (default: %u)", DEFAULT_NAME_LOOKUP), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -457,12 +459,12 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-maxconnections=<n>", strprintf("Maintain at most <n> connections to peers (default: %u). This limit does not apply to connections manually added via -addnode or the addnode RPC, which have a separate limit of %u.", DEFAULT_MAX_PEER_CONNECTIONS, MAX_ADDNODE_CONNECTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxreceivebuffer=<n>", strprintf("Maximum per-connection receive buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXRECEIVEBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxsendbuffer=<n>", strprintf("Maximum per-connection send buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXSENDBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-maxtimeadjustment", strprintf("Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by peers forward or backward by this amount. (default: %u seconds)", DEFAULT_MAX_TIME_ADJUSTMENT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-maxtimeadjustment", strprintf("Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by outbound peers forward or backward by this amount (default: %u seconds).", DEFAULT_MAX_TIME_ADJUSTMENT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxuploadtarget=<n>", strprintf("Tries to keep outbound traffic under the given target per 24h. Limit does not apply to peers with 'download' permission or blocks created within past week. 0 = no limit (default: %s). Optional suffix units [k|K|m|M|g|G|t|T] (default: M). Lowercase is 1000 base while uppercase is 1024 base", DEFAULT_MAX_UPLOAD_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-onion=<ip:port>", "Use separate SOCKS5 proxy to reach peers via Tor onion services, set -noonion to disable (default: -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-i2psam=<ip:port>", "I2P SAM proxy to reach I2P peers and accept I2P connections (default: none)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-i2pacceptincoming", "If set and -i2psam is also set then incoming I2P connections are accepted via the SAM proxy. If this is not set but -i2psam is set then only outgoing connections will be made to the I2P network. Ignored if -i2psam is not set. Listening for incoming I2P connections is done through the SAM proxy, not by binding to a local address and port (default: 1)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-onlynet=<net>", "Make automatic outgoing connections only through network <net> (" + Join(GetNetworkNames(), ", ") + "). Incoming connections are not affected by this option. This option can be specified multiple times to allow multiple networks.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-onlynet=<net>", "Make automatic outbound connections only to network <net> (" + Join(GetNetworkNames(), ", ") + "). Inbound and manual connections are not affected by this option. It can be specified multiple times to allow multiple networks.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerbloomfilters", strprintf("Support filtering of blocks and transaction with bloom filters (default: %u)", DEFAULT_PEERBLOOMFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerblockfilters", strprintf("Serve compact block filters to peers per BIP 157 (default: %u)", DEFAULT_PEERBLOCKFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-permitbaremultisig", strprintf("Relay non-P2SH multisig (default: %u)", DEFAULT_PERMIT_BAREMULTISIG), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -660,7 +662,8 @@ void InitParameterInteraction(ArgsManager& args) LogPrintf("%s: parameter interaction: -connect set -> setting -listen=0\n", __func__); } - if (args.IsArgSet("-proxy")) { + std::string proxy_arg = args.GetArg("-proxy", ""); + if (proxy_arg != "" && proxy_arg != "0") { // to protect privacy, do not listen by default if a default proxy server is specified if (args.SoftSetBoolArg("-listen", false)) LogPrintf("%s: parameter interaction: -proxy set -> setting -listen=0\n", __func__); @@ -792,7 +795,7 @@ bool AppInitBasicSetup(const ArgsManager& args) return true; } -bool AppInitParameterInteraction(const ArgsManager& args) +bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandbox) { const CChainParams& chainparams = Params(); // ********************************************************* Step 2: parameter interactions @@ -853,12 +856,12 @@ bool AppInitParameterInteraction(const ArgsManager& args) nLocalServices = ServiceFlags(nLocalServices | NODE_COMPACT_FILTERS); } - // if using block pruning, then disallow txindex and coinstatsindex if (args.GetIntArg("-prune", 0)) { if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) return InitError(_("Prune mode is incompatible with -txindex.")); - if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) - return InitError(_("Prune mode is incompatible with -coinstatsindex.")); + if (args.GetBoolArg("-reindex-chainstate", false)) { + return InitError(_("Prune mode is incompatible with -reindex-chainstate. Use full -reindex instead.")); + } } // If -forcednsseed is set to true, ensure -dnsseed has not been set to false @@ -1029,6 +1032,19 @@ bool AppInitParameterInteraction(const ArgsManager& args) return InitError(_("No proxy server specified. Use -proxy=<ip> or -proxy=<ip:port>.")); } + if (args.GetBoolArg("-reindex-chainstate", false)) { + // indexes that must be deactivated to prevent index corruption, see #24630 + if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) { + return InitError(_("-reindex-chainstate option is not compatible with -coinstatsindex. Please temporarily disable coinstatsindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.")); + } + if (g_enabled_filter_types.count(BlockFilterType::BASIC)) { + return InitError(_("-reindex-chainstate option is not compatible with -blockfilterindex. Please temporarily disable blockfilterindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.")); + } + if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { + return InitError(_("-reindex-chainstate option is not compatible with -txindex. Please temporarily disable txindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.")); + } + } + #if defined(USE_SYSCALL_SANDBOX) if (args.IsArgSet("-sandbox") && !args.IsArgNegated("-sandbox")) { const std::string sandbox_arg{args.GetArg("-sandbox", "")}; @@ -1056,6 +1072,9 @@ bool AppInitParameterInteraction(const ArgsManager& args) if (!SetupSyscallSandbox(log_syscall_violation_before_terminating)) { return InitError(Untranslated("Installation of the syscall sandbox failed.")); } + if (use_syscall_sandbox) { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION); + } LogPrintf("Experimental syscall sandbox enabled (-sandbox=%s): bitcoind will terminate if an unexpected (not allowlisted) syscall is invoked.\n", sandbox_arg); } #endif // USE_SYSCALL_SANDBOX @@ -1223,16 +1242,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) const bool ignores_incoming_txs{args.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY)}; { - // Initialize addrman - assert(!node.addrman); // Read asmap file if configured std::vector<bool> asmap; if (args.IsArgSet("-asmap")) { - fs::path asmap_path = fs::PathFromString(args.GetArg("-asmap", "")); - if (asmap_path.empty()) { - asmap_path = fs::PathFromString(DEFAULT_ASMAP_FILENAME); - } + fs::path asmap_path = args.GetPathArg("-asmap", DEFAULT_ASMAP_FILENAME); if (!asmap_path.is_absolute()) { asmap_path = gArgs.GetDataDirNet() / asmap_path; } @@ -1251,8 +1265,14 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) LogPrintf("Using /16 prefix for IP bucketing\n"); } + // Initialize netgroup manager + assert(!node.netgroupman); + node.netgroupman = std::make_unique<NetGroupManager>(std::move(asmap)); + + // Initialize addrman + assert(!node.addrman); uiInterface.InitMessage(_("Loading P2P addresses…").translated); - if (const auto error{LoadAddrman(asmap, args, node.addrman)}) { + if (const auto error{LoadAddrman(*node.netgroupman, args, node.addrman)}) { return InitError(*error); } } @@ -1260,26 +1280,15 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) assert(!node.banman); node.banman = std::make_unique<BanMan>(gArgs.GetDataDirNet() / "banlist", &uiInterface, args.GetIntArg("-bantime", DEFAULT_MISBEHAVING_BANTIME)); assert(!node.connman); - node.connman = std::make_unique<CConnman>(GetRand(std::numeric_limits<uint64_t>::max()), GetRand(std::numeric_limits<uint64_t>::max()), *node.addrman, args.GetBoolArg("-networkactive", true)); + node.connman = std::make_unique<CConnman>(GetRand(std::numeric_limits<uint64_t>::max()), + GetRand(std::numeric_limits<uint64_t>::max()), + *node.addrman, *node.netgroupman, args.GetBoolArg("-networkactive", true)); assert(!node.fee_estimator); // Don't initialize fee estimation with old data if we don't relay transactions, // as they would never get updated. if (!ignores_incoming_txs) node.fee_estimator = std::make_unique<CBlockPolicyEstimator>(); - assert(!node.mempool); - int check_ratio = std::min<int>(std::max<int>(args.GetIntArg("-checkmempool", chainparams.DefaultConsistencyChecks() ? 1 : 0), 0), 1000000); - node.mempool = std::make_unique<CTxMemPool>(node.fee_estimator.get(), check_ratio); - - assert(!node.chainman); - node.chainman = std::make_unique<ChainstateManager>(); - ChainstateManager& chainman = *node.chainman; - - assert(!node.peerman); - node.peerman = PeerManager::make(chainparams, *node.connman, *node.addrman, node.banman.get(), - chainman, *node.mempool, ignores_incoming_txs); - RegisterValidationInterface(node.peerman.get()); - // sanitize comments per BIP-0014, format user agent and check total size std::vector<std::string> uacomments; for (const std::string& cmt : args.GetArgs("-uacomment")) { @@ -1303,6 +1312,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } for (int n = 0; n < NET_MAX; n++) { enum Network net = (enum Network)n; + assert(IsReachable(net)); if (!nets.count(net)) SetReachable(net, false); } @@ -1407,8 +1417,16 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) LogPrintf("* Using %.1f MiB for chain state database\n", cache_sizes.coins_db * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", cache_sizes.coins * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024)); - bool fLoaded = false; - while (!fLoaded && !ShutdownRequested()) { + assert(!node.mempool); + assert(!node.chainman); + const int mempool_check_ratio = std::clamp<int>(args.GetIntArg("-checkmempool", chainparams.DefaultConsistencyChecks() ? 1 : 0), 0, 1000000); + + for (bool fLoaded = false; !fLoaded && !ShutdownRequested();) { + node.mempool = std::make_unique<CTxMemPool>(node.fee_estimator.get(), mempool_check_ratio); + + node.chainman = std::make_unique<ChainstateManager>(); + ChainstateManager& chainman = *node.chainman; + const bool fReset = fReindex; bilingual_str strLoadError; @@ -1453,8 +1471,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) strLoadError = _("Error initializing block database"); break; case ChainstateLoadingError::ERROR_CHAINSTATE_UPGRADE_FAILED: - strLoadError = _("Error upgrading chainstate database"); - break; + return InitError(_("Unsupported chainstate database format found. " + "Please restart with -reindex-chainstate. This will " + "rebuild the chainstate database.")); case ChainstateLoadingError::ERROR_REPLAYBLOCKS_FAILED: strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate."); break; @@ -1476,7 +1495,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) try { uiInterface.InitMessage(_("Verifying blocks…").translated); auto check_blocks = args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); - if (fHavePruned && check_blocks > MIN_BLOCKS_TO_KEEP) { + if (chainman.m_blockman.m_have_pruned && check_blocks > MIN_BLOCKS_TO_KEEP) { LogPrintf("Prune: pruned datadir may not have more than %d blocks; only checking available blocks\n", MIN_BLOCKS_TO_KEEP); } @@ -1539,9 +1558,16 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) return false; } + ChainstateManager& chainman = *Assert(node.chainman); + + assert(!node.peerman); + node.peerman = PeerManager::make(chainparams, *node.connman, *node.addrman, node.banman.get(), + chainman, *node.mempool, ignores_incoming_txs); + RegisterValidationInterface(node.peerman.get()); + // ********************************************************* Step 8: start indexers if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { - if (const auto error{CheckLegacyTxindex(*Assert(chainman.m_blockman.m_block_tree_db))}) { + if (const auto error{WITH_LOCK(cs_main, return CheckLegacyTxindex(*Assert(chainman.m_blockman.m_block_tree_db)))}) { return InitError(*error); } @@ -1614,7 +1640,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) uiInterface.NotifyBlockTip_connect([block_notify](SynchronizationState sync_state, const CBlockIndex* pBlockIndex) { if (sync_state != SynchronizationState::POST_INIT || !pBlockIndex) return; std::string command = block_notify; - boost::replace_all(command, "%s", pBlockIndex->GetBlockHash().GetHex()); + ReplaceAll(command, "%s", pBlockIndex->GetBlockHash().GetHex()); std::thread t(runCommand, command); t.detach(); // thread runs free }); @@ -1660,9 +1686,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) tip_info->block_time = chainman.ActiveChain().Tip() ? chainman.ActiveChain().Tip()->GetBlockTime() : Params().GenesisBlock().GetBlockTime(); tip_info->verification_progress = GuessVerificationProgress(Params().TxData(), chainman.ActiveChain().Tip()); } - if (tip_info && ::pindexBestHeader) { - tip_info->header_height = ::pindexBestHeader->nHeight; - tip_info->header_time = ::pindexBestHeader->GetBlockTime(); + if (tip_info && chainman.m_best_header) { + tip_info->header_height = chainman.m_best_header->nHeight; + tip_info->header_time = chainman.m_best_header->GetBlockTime(); } } LogPrintf("nBestHeight = %d\n", chain_active_height); diff --git a/src/init.h b/src/init.h index ddd439f619..2250ae20a0 100644 --- a/src/init.h +++ b/src/init.h @@ -41,7 +41,7 @@ bool AppInitBasicSetup(const ArgsManager& args); * @note This can be done before daemonization. Do not call Shutdown() if this function fails. * @pre Parameters should be parsed and config file should be read, AppInitBasicSetup should have been called. */ -bool AppInitParameterInteraction(const ArgsManager& args); +bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandbox = true); /** * Initialization sanity checks: ecc init, sanity checks, dir lock. * @note This can be done before daemonization. Do not call Shutdown() if this function fails. diff --git a/src/init/bitcoin-gui.cpp b/src/init/bitcoin-gui.cpp index e297b48718..2fa4add4e5 100644 --- a/src/init/bitcoin-gui.cpp +++ b/src/init/bitcoin-gui.cpp @@ -9,6 +9,7 @@ #include <interfaces/node.h> #include <interfaces/wallet.h> #include <node/context.h> +#include <util/check.h> #include <util/system.h> #include <memory> diff --git a/src/init/bitcoin-node.cpp b/src/init/bitcoin-node.cpp index 511a872bc0..78bc3e5980 100644 --- a/src/init/bitcoin-node.cpp +++ b/src/init/bitcoin-node.cpp @@ -9,6 +9,7 @@ #include <interfaces/node.h> #include <interfaces/wallet.h> #include <node/context.h> +#include <util/check.h> #include <util/system.h> #include <memory> diff --git a/src/init/bitcoin-qt.cpp b/src/init/bitcoin-qt.cpp index 37d4e426c5..bb3bb945d0 100644 --- a/src/init/bitcoin-qt.cpp +++ b/src/init/bitcoin-qt.cpp @@ -8,6 +8,7 @@ #include <interfaces/node.h> #include <interfaces/wallet.h> #include <node/context.h> +#include <util/check.h> #include <util/system.h> #include <memory> diff --git a/src/init/bitcoin-wallet.cpp b/src/init/bitcoin-wallet.cpp index e9dcde72fe..c8d499da10 100644 --- a/src/init/bitcoin-wallet.cpp +++ b/src/init/bitcoin-wallet.cpp @@ -4,6 +4,8 @@ #include <interfaces/init.h> +#include <memory> + namespace interfaces { std::unique_ptr<Init> MakeWalletInit(int argc, char* argv[], int& exit_status) { diff --git a/src/init/bitcoind.cpp b/src/init/bitcoind.cpp index 2addff07c1..b473ebb805 100644 --- a/src/init/bitcoind.cpp +++ b/src/init/bitcoind.cpp @@ -8,6 +8,7 @@ #include <interfaces/node.h> #include <interfaces/wallet.h> #include <node/context.h> +#include <util/check.h> #include <util/system.h> #include <memory> diff --git a/src/init/common.cpp b/src/init/common.cpp index 38c60366e3..eac6732968 100644 --- a/src/init/common.cpp +++ b/src/init/common.cpp @@ -9,16 +9,21 @@ #include <clientversion.h> #include <compat/sanity.h> #include <crypto/sha256.h> +#include <fs.h> #include <key.h> #include <logging.h> #include <node/ui_interface.h> #include <pubkey.h> #include <random.h> +#include <tinyformat.h> #include <util/system.h> #include <util/time.h> #include <util/translation.h> +#include <algorithm> #include <memory> +#include <string> +#include <vector> static std::unique_ptr<ECCVerifyHandle> globalVerifyHandle; @@ -81,7 +86,7 @@ void AddLoggingArgs(ArgsManager& argsman) void SetLoggingOptions(const ArgsManager& args) { LogInstance().m_print_to_file = !args.IsArgNegated("-debuglogfile"); - LogInstance().m_file_path = AbsPathForConfigVal(fs::PathFromString(args.GetArg("-debuglogfile", DEFAULT_DEBUGLOGFILE))); + LogInstance().m_file_path = AbsPathForConfigVal(args.GetPathArg("-debuglogfile", DEFAULT_DEBUGLOGFILE)); LogInstance().m_print_to_console = args.GetBoolArg("-printtoconsole", !args.GetBoolArg("-daemon", false)); LogInstance().m_log_timestamps = args.GetBoolArg("-logtimestamps", DEFAULT_LOGTIMESTAMPS); LogInstance().m_log_time_micros = args.GetBoolArg("-logtimemicros", DEFAULT_LOGTIMEMICROS); diff --git a/src/kernel/bitcoinkernel.cpp b/src/kernel/bitcoinkernel.cpp new file mode 100644 index 0000000000..bb101ba186 --- /dev/null +++ b/src/kernel/bitcoinkernel.cpp @@ -0,0 +1,10 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <functional> +#include <string> + +// Define G_TRANSLATION_FUN symbol in libbitcoinkernel library so users of the +// library aren't required to export this symbol +extern const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; diff --git a/src/key.cpp b/src/key.cpp index 354bd097ce..d1d521f97d 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -159,7 +159,7 @@ bool CKey::Check(const unsigned char *vch) { void CKey::MakeNewKey(bool fCompressedIn) { do { - GetStrongRandBytes(keydata.data(), keydata.size()); + GetStrongRandBytes(keydata); } while (!Check(keydata.data())); fValid = true; fCompressed = fCompressedIn; @@ -244,7 +244,7 @@ bool CKey::VerifyPubKey(const CPubKey& pubkey) const { } unsigned char rnd[8]; std::string str = "Bitcoin key verification\n"; - GetRandBytes(rnd, sizeof(rnd)); + GetRandBytes(rnd); uint256 hash; CHash256().Write(MakeUCharSpan(str)).Write(rnd).Finalize(hash); std::vector<unsigned char> vchSig; @@ -288,7 +288,7 @@ bool CKey::SignSchnorr(const uint256& hash, Span<unsigned char> sig, const uint2 uint256 tweak = XOnlyPubKey(pubkey_bytes).ComputeTapTweakHash(merkle_root->IsNull() ? nullptr : merkle_root); if (!secp256k1_keypair_xonly_tweak_add(GetVerifyContext(), &keypair, tweak.data())) return false; } - bool ret = secp256k1_schnorrsig_sign(secp256k1_context_sign, sig.data(), hash.data(), &keypair, aux.data()); + bool ret = secp256k1_schnorrsig_sign32(secp256k1_context_sign, sig.data(), hash.data(), &keypair, aux.data()); if (ret) { // Additional verification step to prevent using a potentially corrupted signature secp256k1_xonly_pubkey pubkey_verify; @@ -397,7 +397,7 @@ void ECC_Start() { { // Pass in a random blinding seed to the secp256k1 context. std::vector<unsigned char, secure_allocator<unsigned char>> vseed(32); - GetRandBytes(vseed.data(), 32); + GetRandBytes(vseed); bool ret = secp256k1_context_randomize(ctx, vseed.data()); assert(ret); } diff --git a/src/leveldb/util/env_posix.cc b/src/leveldb/util/env_posix.cc index 9f5863a0f3..18626b327c 100644 --- a/src/leveldb/util/env_posix.cc +++ b/src/leveldb/util/env_posix.cc @@ -850,7 +850,7 @@ class SingletonEnv { public: SingletonEnv() { #if !defined(NDEBUG) - env_initialized_.store(true, std::memory_order::memory_order_relaxed); + env_initialized_.store(true, std::memory_order_relaxed); #endif // !defined(NDEBUG) static_assert(sizeof(env_storage_) >= sizeof(EnvType), "env_storage_ will not fit the Env"); @@ -867,7 +867,7 @@ class SingletonEnv { static void AssertEnvNotInitialized() { #if !defined(NDEBUG) - assert(!env_initialized_.load(std::memory_order::memory_order_relaxed)); + assert(!env_initialized_.load(std::memory_order_relaxed)); #endif // !defined(NDEBUG) } diff --git a/src/leveldb/util/env_windows.cc b/src/leveldb/util/env_windows.cc index 1834206562..4dcba222a1 100644 --- a/src/leveldb/util/env_windows.cc +++ b/src/leveldb/util/env_windows.cc @@ -798,7 +798,7 @@ class SingletonEnv { public: SingletonEnv() { #if !defined(NDEBUG) - env_initialized_.store(true, std::memory_order::memory_order_relaxed); + env_initialized_.store(true, std::memory_order_relaxed); #endif // !defined(NDEBUG) static_assert(sizeof(env_storage_) >= sizeof(EnvType), "env_storage_ will not fit the Env"); @@ -815,7 +815,7 @@ class SingletonEnv { static void AssertEnvNotInitialized() { #if !defined(NDEBUG) - assert(!env_initialized_.load(std::memory_order::memory_order_relaxed)); + assert(!env_initialized_.load(std::memory_order_relaxed)); #endif // !defined(NDEBUG) } diff --git a/src/logging.cpp b/src/logging.cpp index 764941c8ea..a5e5fdb4cd 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -160,7 +160,9 @@ const CLogCategoryDesc LogCategories[] = {BCLog::VALIDATION, "validation"}, {BCLog::I2P, "i2p"}, {BCLog::IPC, "ipc"}, +#ifdef DEBUG_LOCKCONTENTION {BCLog::LOCK, "lock"}, +#endif {BCLog::UTIL, "util"}, {BCLog::BLOCKSTORE, "blockstorage"}, {BCLog::ALL, "1"}, diff --git a/src/logging.h b/src/logging.h index 710e6c4c32..ae8cad906c 100644 --- a/src/logging.h +++ b/src/logging.h @@ -60,7 +60,9 @@ namespace BCLog { VALIDATION = (1 << 21), I2P = (1 << 22), IPC = (1 << 23), +#ifdef DEBUG_LOCKCONTENTION LOCK = (1 << 24), +#endif UTIL = (1 << 25), BLOCKSTORE = (1 << 26), ALL = ~(uint32_t)0, diff --git a/src/logging/timer.h b/src/logging/timer.h index fc5307bc62..d954e46301 100644 --- a/src/logging/timer.h +++ b/src/logging/timer.h @@ -97,13 +97,13 @@ private: #define LOG_TIME_MICROS_WITH_CATEGORY(end_msg, log_category) \ - BCLog::Timer<std::chrono::microseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, log_category) + BCLog::Timer<std::chrono::microseconds> UNIQUE_NAME(logging_timer)(__func__, end_msg, log_category) #define LOG_TIME_MILLIS_WITH_CATEGORY(end_msg, log_category) \ - BCLog::Timer<std::chrono::milliseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, log_category) + BCLog::Timer<std::chrono::milliseconds> UNIQUE_NAME(logging_timer)(__func__, end_msg, log_category) #define LOG_TIME_MILLIS_WITH_CATEGORY_MSG_ONCE(end_msg, log_category) \ - BCLog::Timer<std::chrono::milliseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, log_category, /* msg_on_completion=*/false) + BCLog::Timer<std::chrono::milliseconds> UNIQUE_NAME(logging_timer)(__func__, end_msg, log_category, /* msg_on_completion=*/false) #define LOG_TIME_SECONDS(end_msg) \ - BCLog::Timer<std::chrono::seconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg) + BCLog::Timer<std::chrono::seconds> UNIQUE_NAME(logging_timer)(__func__, end_msg) #endif // BITCOIN_LOGGING_TIMER_H diff --git a/src/mapport.cpp b/src/mapport.cpp index 42ca366089..235e6f904c 100644 --- a/src/mapport.cpp +++ b/src/mapport.cpp @@ -193,7 +193,7 @@ static bool ProcessUpnp() std::string strDesc = PACKAGE_NAME " " + FormatFullVersion(); do { - r = UPNP_AddPortMapping(urls.controlURL, data.first.servicetype, port.c_str(), port.c_str(), lanaddr, strDesc.c_str(), "TCP", 0, "0"); + r = UPNP_AddPortMapping(urls.controlURL, data.first.servicetype, port.c_str(), port.c_str(), lanaddr, strDesc.c_str(), "TCP", nullptr, "0"); if (r != UPNPCOMMAND_SUCCESS) { ret = false; @@ -206,7 +206,7 @@ static bool ProcessUpnp() } while (g_mapport_interrupt.sleep_for(PORT_MAPPING_REANNOUNCE_PERIOD)); g_mapport_interrupt.reset(); - r = UPNP_DeletePortMapping(urls.controlURL, data.first.servicetype, port.c_str(), "TCP", 0); + r = UPNP_DeletePortMapping(urls.controlURL, data.first.servicetype, port.c_str(), "TCP", nullptr); LogPrintf("UPNP_DeletePortMapping() returned: %d\n", r); freeUPNPDevlist(devlist); devlist = nullptr; FreeUPNPUrls(&urls); diff --git a/src/net.cpp b/src/net.cpp index 955eec46e3..46d7020c5e 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -103,7 +103,7 @@ enum BindFlags { // The sleep time needs to be small to avoid new sockets stalling static const uint64_t SELECT_TIMEOUT_MILLISECONDS = 50; -const std::string NET_MESSAGE_COMMAND_OTHER = "*other*"; +const std::string NET_MESSAGE_TYPE_OTHER = "*other*"; static const uint64_t RANDOMIZER_ID_NETGROUP = 0x6c0edd8036ef4036ULL; // SHA256("netgroup")[0:8] static const uint64_t RANDOMIZER_ID_LOCALHOSTNONCE = 0xd93e69e2bbfa5735ULL; // SHA256("localhostnonce")[0:8] @@ -626,12 +626,6 @@ void CNode::CopyStats(CNodeStats& stats) X(addr); X(addrBind); stats.m_network = ConnectedThroughNetwork(); - if (m_tx_relay != nullptr) { - LOCK(m_tx_relay->cs_filter); - stats.fRelayTxes = m_tx_relay->fRelayTxes; - } else { - stats.fRelayTxes = false; - } X(m_last_send); X(m_last_recv); X(m_last_tx_time); @@ -649,20 +643,15 @@ void CNode::CopyStats(CNodeStats& stats) X(m_bip152_highbandwidth_from); { LOCK(cs_vSend); - X(mapSendBytesPerMsgCmd); + X(mapSendBytesPerMsgType); X(nSendBytes); } { LOCK(cs_vRecv); - X(mapRecvBytesPerMsgCmd); + X(mapRecvBytesPerMsgType); X(nRecvBytes); } X(m_permissionFlags); - if (m_tx_relay != nullptr) { - stats.minFeeFilter = m_tx_relay->minFeeFilter; - } else { - stats.minFeeFilter = 0; - } X(m_last_ping_time); X(m_min_ping_time); @@ -695,19 +684,19 @@ bool CNode::ReceiveMsgBytes(Span<const uint8_t> msg_bytes, bool& complete) bool reject_message{false}; CNetMessage msg = m_deserializer->GetMessage(time, reject_message); if (reject_message) { - // Message deserialization failed. Drop the message but don't disconnect the peer. + // Message deserialization failed. Drop the message but don't disconnect the peer. // store the size of the corrupt message - mapRecvBytesPerMsgCmd.at(NET_MESSAGE_COMMAND_OTHER) += msg.m_raw_message_size; + mapRecvBytesPerMsgType.at(NET_MESSAGE_TYPE_OTHER) += msg.m_raw_message_size; continue; } - // Store received bytes per message command - // to prevent a memory DOS, only allow valid commands - auto i = mapRecvBytesPerMsgCmd.find(msg.m_type); - if (i == mapRecvBytesPerMsgCmd.end()) { - i = mapRecvBytesPerMsgCmd.find(NET_MESSAGE_COMMAND_OTHER); + // Store received bytes per message type. + // To prevent a memory DOS, only allow known message types. + auto i = mapRecvBytesPerMsgType.find(msg.m_type); + if (i == mapRecvBytesPerMsgType.end()) { + i = mapRecvBytesPerMsgType.find(NET_MESSAGE_TYPE_OTHER); } - assert(i != mapRecvBytesPerMsgCmd.end()); + assert(i != mapRecvBytesPerMsgType.end()); i->second += msg.m_raw_message_size; // push the message to the process queue, @@ -792,7 +781,7 @@ CNetMessage V1TransportDeserializer::GetMessage(const std::chrono::microseconds // decompose a single CNetMessage from the TransportDeserializer CNetMessage msg(std::move(vRecv)); - // store command string, time, and sizes + // store message type string, time, and sizes msg.m_type = hdr.GetCommand(); msg.m_time = time; msg.m_message_size = hdr.nMessageSize; @@ -803,7 +792,7 @@ CNetMessage V1TransportDeserializer::GetMessage(const std::chrono::microseconds // We just received a message off the wire, harvest entropy from the time (and the message checksum) RandAddEvent(ReadLE32(hash.begin())); - // Check checksum and header command string + // Check checksum and header message type string if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) { LogPrint(BCLog::NET, "Header error: Wrong checksum (%s, %u bytes), expected %s was %s, peer=%d\n", SanitizeString(msg.m_type), msg.m_message_size, @@ -913,7 +902,7 @@ static bool CompareNodeTXTime(const NodeEvictionCandidate &a, const NodeEviction { // There is a fall-through here because it is common for a node to have more than a few peers that have not yet relayed txn. if (a.m_last_tx_time != b.m_last_tx_time) return a.m_last_tx_time < b.m_last_tx_time; - if (a.fRelayTxes != b.fRelayTxes) return b.fRelayTxes; + if (a.m_relay_txs != b.m_relay_txs) return b.m_relay_txs; if (a.fBloomFilter != b.fBloomFilter) return a.fBloomFilter; return a.m_connected > b.m_connected; } @@ -921,7 +910,7 @@ static bool CompareNodeTXTime(const NodeEvictionCandidate &a, const NodeEviction // Pick out the potential block-relay only peers, and sort them by last block time. static bool CompareNodeBlockRelayOnlyTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) { - if (a.fRelayTxes != b.fRelayTxes) return a.fRelayTxes; + if (a.m_relay_txs != b.m_relay_txs) return a.m_relay_txs; if (a.m_last_block_time != b.m_last_block_time) return a.m_last_block_time < b.m_last_block_time; if (a.fRelevantServices != b.fRelevantServices) return b.fRelevantServices; return a.m_connected > b.m_connected; @@ -1046,7 +1035,7 @@ void ProtectEvictionCandidatesByRatio(std::vector<NodeEvictionCandidate>& evicti EraseLastKElements(vEvictionCandidates, CompareNodeTXTime, 4); // Protect up to 8 non-tx-relay peers that have sent us novel blocks. EraseLastKElements(vEvictionCandidates, CompareNodeBlockRelayOnlyTime, 8, - [](const NodeEvictionCandidate& n) { return !n.fRelayTxes && n.fRelevantServices; }); + [](const NodeEvictionCandidate& n) { return !n.m_relay_txs && n.fRelevantServices; }); // Protect 4 nodes that most recently sent us novel blocks. // An attacker cannot manipulate this metric without performing useful work. @@ -1112,18 +1101,11 @@ bool CConnman::AttemptToEvictConnection() continue; if (node->fDisconnect) continue; - bool peer_relay_txes = false; - bool peer_filter_not_null = false; - if (node->m_tx_relay != nullptr) { - LOCK(node->m_tx_relay->cs_filter); - peer_relay_txes = node->m_tx_relay->fRelayTxes; - peer_filter_not_null = node->m_tx_relay->pfilter != nullptr; - } NodeEvictionCandidate candidate = {node->GetId(), node->m_connected, node->m_min_ping_time, node->m_last_block_time, node->m_last_tx_time, HasAllDesirableServiceFlags(node->nServices), - peer_relay_txes, peer_filter_not_null, node->nKeyedNetGroup, - node->m_prefer_evict, node->addr.IsLocal(), + node->m_relays_txs.load(), node->m_bloom_filter_loaded.load(), + node->nKeyedNetGroup, node->m_prefer_evict, node->addr.IsLocal(), node->ConnectedThroughNetwork()}; vEvictionCandidates.push_back(candidate); } @@ -1208,7 +1190,11 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, // According to the internet TCP_NODELAY is not carried into accepted sockets // on all platforms. Set it again here just to be sure. - SetSocketNoDelay(sock->Get()); + const int on{1}; + if (sock->SetSockOpt(IPPROTO_TCP, TCP_NODELAY, &on, sizeof(on)) == SOCKET_ERROR) { + LogPrint(BCLog::NET, "connection from %s: unable to set TCP_NODELAY, continuing anyway\n", + addr.ToString()); + } // Don't accept connections from banned peers. bool banned = m_banman && m_banman->IsBanned(addr); @@ -1581,6 +1567,8 @@ void CConnman::SocketEvents(const std::vector<CNode*>& nodes, void CConnman::SocketHandler() { + AssertLockNotHeld(m_total_bytes_sent_mutex); + std::set<SOCKET> recv_set; std::set<SOCKET> send_set; std::set<SOCKET> error_set; @@ -1607,6 +1595,8 @@ void CConnman::SocketHandlerConnected(const std::vector<CNode*>& nodes, const std::set<SOCKET>& send_set, const std::set<SOCKET>& error_set) { + AssertLockNotHeld(m_total_bytes_sent_mutex); + for (CNode* pnode : nodes) { if (interruptNet) return; @@ -1708,6 +1698,8 @@ void CConnman::SocketHandlerListening(const std::set<SOCKET>& recv_set) void CConnman::ThreadSocketHandler() { + AssertLockNotHeld(m_total_bytes_sent_mutex); + SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET); while (!interruptNet) { @@ -2015,7 +2007,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) case ConnectionType::BLOCK_RELAY: case ConnectionType::ADDR_FETCH: case ConnectionType::FEELER: - setConnected.insert(pnode->addr.GetGroup(addrman.GetAsmap())); + setConnected.insert(m_netgroupman.GetGroup(pnode->addr)); } // no default case, so the compiler can warn about missing cases } } @@ -2089,7 +2081,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) m_anchors.pop_back(); if (!addr.IsValid() || IsLocal(addr) || !IsReachable(addr) || !HasAllDesirableServiceFlags(addr.nServices) || - setConnected.count(addr.GetGroup(addrman.GetAsmap()))) continue; + setConnected.count(m_netgroupman.GetGroup(addr))) continue; addrConnect = addr; LogPrint(BCLog::NET, "Trying to make an anchor connection to %s\n", addrConnect.ToString()); break; @@ -2130,7 +2122,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) } // Require outbound connections, other than feelers, to be to distinct network groups - if (!fFeeler && setConnected.count(addr.GetGroup(addrman.GetAsmap()))) { + if (!fFeeler && setConnected.count(m_netgroupman.GetGroup(addr))) { break; } @@ -2413,17 +2405,26 @@ bool CConnman::BindListenPort(const CService& addrBind, bilingual_str& strError, // Allow binding if the port is still in TIME_WAIT state after // the program was closed and restarted. - setsockopt(sock->Get(), SOL_SOCKET, SO_REUSEADDR, (sockopt_arg_type)&nOne, sizeof(int)); + if (sock->SetSockOpt(SOL_SOCKET, SO_REUSEADDR, (sockopt_arg_type)&nOne, sizeof(int)) == SOCKET_ERROR) { + strError = strprintf(Untranslated("Error setting SO_REUSEADDR on socket: %s, continuing anyway"), NetworkErrorString(WSAGetLastError())); + LogPrintf("%s\n", strError.original); + } // some systems don't have IPV6_V6ONLY but are always v6only; others do have the option // and enable it by default or not. Try to enable it, if possible. if (addrBind.IsIPv6()) { #ifdef IPV6_V6ONLY - setsockopt(sock->Get(), IPPROTO_IPV6, IPV6_V6ONLY, (sockopt_arg_type)&nOne, sizeof(int)); + if (sock->SetSockOpt(IPPROTO_IPV6, IPV6_V6ONLY, (sockopt_arg_type)&nOne, sizeof(int)) == SOCKET_ERROR) { + strError = strprintf(Untranslated("Error setting IPV6_V6ONLY on socket: %s, continuing anyway"), NetworkErrorString(WSAGetLastError())); + LogPrintf("%s\n", strError.original); + } #endif #ifdef WIN32 int nProtLevel = PROTECTION_LEVEL_UNRESTRICTED; - setsockopt(sock->Get(), IPPROTO_IPV6, IPV6_PROTECTION_LEVEL, (const char*)&nProtLevel, sizeof(int)); + if (sock->SetSockOpt(IPPROTO_IPV6, IPV6_PROTECTION_LEVEL, (const char*)&nProtLevel, sizeof(int)) == SOCKET_ERROR) { + strError = strprintf(Untranslated("Error setting IPV6_PROTECTION_LEVEL on socket: %s, continuing anyway"), NetworkErrorString(WSAGetLastError())); + LogPrintf("%s\n", strError.original); + } #endif } @@ -2517,8 +2518,12 @@ void CConnman::SetNetworkActive(bool active) } } -CConnman::CConnman(uint64_t nSeed0In, uint64_t nSeed1In, AddrMan& addrman_in, bool network_active) - : addrman(addrman_in), nSeed0(nSeed0In), nSeed1(nSeed1In) +CConnman::CConnman(uint64_t nSeed0In, uint64_t nSeed1In, AddrMan& addrman_in, + const NetGroupManager& netgroupman, bool network_active) + : addrman(addrman_in) + , m_netgroupman{netgroupman} + , nSeed0(nSeed0In) + , nSeed1(nSeed1In) { SetTryNewOutboundPeer(false); @@ -2579,6 +2584,7 @@ bool CConnman::InitBinds(const Options& options) bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) { + AssertLockNotHeld(m_total_bytes_sent_mutex); Init(connOptions); if (fListen && !InitBinds(connOptions)) { @@ -2800,7 +2806,7 @@ std::vector<CAddress> CConnman::GetAddresses(CNode& requestor, size_t max_addres auto r = m_addr_response_caches.emplace(cache_id, CachedAddrResponse{}); CachedAddrResponse& cache_entry = r.first->second; if (cache_entry.m_cache_entry_expiration < current_time) { // If emplace() added new one it has expiration 0. - cache_entry.m_addrs_response_cache = GetAddresses(max_addresses, max_pct, /* network */ std::nullopt); + cache_entry.m_addrs_response_cache = GetAddresses(max_addresses, max_pct, /*network=*/std::nullopt); // Choosing a proper cache lifetime is a trade-off between the privacy leak minimization // and the usefulness of ADDR responses to honest users. // @@ -2877,7 +2883,7 @@ void CConnman::GetNodeStats(std::vector<CNodeStats>& vstats) const for (CNode* pnode : m_nodes) { vstats.emplace_back(); pnode->CopyStats(vstats.back()); - vstats.back().m_mapped_as = pnode->addr.GetMappedAS(addrman.GetAsmap()); + vstats.back().m_mapped_as = m_netgroupman.GetMappedAS(pnode->addr); } } @@ -2931,7 +2937,9 @@ void CConnman::RecordBytesRecv(uint64_t bytes) void CConnman::RecordBytesSent(uint64_t bytes) { - LOCK(cs_totalBytesSent); + AssertLockNotHeld(m_total_bytes_sent_mutex); + LOCK(m_total_bytes_sent_mutex); + nTotalBytesSent += bytes; const auto now = GetTime<std::chrono::seconds>(); @@ -2947,7 +2955,8 @@ void CConnman::RecordBytesSent(uint64_t bytes) uint64_t CConnman::GetMaxOutboundTarget() const { - LOCK(cs_totalBytesSent); + AssertLockNotHeld(m_total_bytes_sent_mutex); + LOCK(m_total_bytes_sent_mutex); return nMaxOutboundLimit; } @@ -2958,7 +2967,15 @@ std::chrono::seconds CConnman::GetMaxOutboundTimeframe() const std::chrono::seconds CConnman::GetMaxOutboundTimeLeftInCycle() const { - LOCK(cs_totalBytesSent); + AssertLockNotHeld(m_total_bytes_sent_mutex); + LOCK(m_total_bytes_sent_mutex); + return GetMaxOutboundTimeLeftInCycle_(); +} + +std::chrono::seconds CConnman::GetMaxOutboundTimeLeftInCycle_() const +{ + AssertLockHeld(m_total_bytes_sent_mutex); + if (nMaxOutboundLimit == 0) return 0s; @@ -2972,14 +2989,15 @@ std::chrono::seconds CConnman::GetMaxOutboundTimeLeftInCycle() const bool CConnman::OutboundTargetReached(bool historicalBlockServingLimit) const { - LOCK(cs_totalBytesSent); + AssertLockNotHeld(m_total_bytes_sent_mutex); + LOCK(m_total_bytes_sent_mutex); if (nMaxOutboundLimit == 0) return false; if (historicalBlockServingLimit) { // keep a large enough buffer to at least relay each block once - const std::chrono::seconds timeLeftInCycle = GetMaxOutboundTimeLeftInCycle(); + const std::chrono::seconds timeLeftInCycle = GetMaxOutboundTimeLeftInCycle_(); const uint64_t buffer = timeLeftInCycle / std::chrono::minutes{10} * MAX_BLOCK_SERIALIZED_SIZE; if (buffer >= nMaxOutboundLimit || nMaxOutboundTotalBytesSentInCycle >= nMaxOutboundLimit - buffer) return true; @@ -2992,7 +3010,8 @@ bool CConnman::OutboundTargetReached(bool historicalBlockServingLimit) const uint64_t CConnman::GetOutboundTargetBytesLeft() const { - LOCK(cs_totalBytesSent); + AssertLockNotHeld(m_total_bytes_sent_mutex); + LOCK(m_total_bytes_sent_mutex); if (nMaxOutboundLimit == 0) return 0; @@ -3006,7 +3025,8 @@ uint64_t CConnman::GetTotalBytesRecv() const uint64_t CConnman::GetTotalBytesSent() const { - LOCK(cs_totalBytesSent); + AssertLockNotHeld(m_total_bytes_sent_mutex); + LOCK(m_total_bytes_sent_mutex); return nTotalBytesSent; } @@ -3031,13 +3051,10 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, std::shared_ptr<Sock> s nLocalServices(nLocalServicesIn) { if (inbound_onion) assert(conn_type_in == ConnectionType::INBOUND); - if (conn_type_in != ConnectionType::BLOCK_RELAY) { - m_tx_relay = std::make_unique<TxRelay>(); - } for (const std::string &msg : getAllNetMessageTypes()) - mapRecvBytesPerMsgCmd[msg] = 0; - mapRecvBytesPerMsgCmd[NET_MESSAGE_COMMAND_OTHER] = 0; + mapRecvBytesPerMsgType[msg] = 0; + mapRecvBytesPerMsgType[NET_MESSAGE_TYPE_OTHER] = 0; if (fLogIPs) { LogPrint(BCLog::NET, "Added connection to %s peer=%d\n", m_addr_name, id); @@ -3056,6 +3073,7 @@ bool CConnman::NodeFullyConnected(const CNode* pnode) void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) { + AssertLockNotHeld(m_total_bytes_sent_mutex); size_t nMessageSize = msg.data.size(); LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", msg.m_type, nMessageSize, pnode->GetId()); if (gArgs.GetBoolArg("-capturemessages", false)) { @@ -3082,7 +3100,7 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) bool optimisticSend(pnode->vSendMsg.empty()); //log total amount of bytes per message type - pnode->mapSendBytesPerMsgCmd[msg.m_type] += nTotalSize; + pnode->mapSendBytesPerMsgType[msg.m_type] += nTotalSize; pnode->nSendSize += nTotalSize; if (pnode->nSendSize > nSendBufferMaxSize) pnode->fPauseSend = true; @@ -3113,9 +3131,9 @@ CSipHasher CConnman::GetDeterministicRandomizer(uint64_t id) const return CSipHasher(nSeed0, nSeed1).Write(id); } -uint64_t CConnman::CalculateKeyedNetGroup(const CAddress& ad) const +uint64_t CConnman::CalculateKeyedNetGroup(const CAddress& address) const { - std::vector<unsigned char> vchNetGroup(ad.GetGroup(addrman.GetAsmap())); + std::vector<unsigned char> vchNetGroup(m_netgroupman.GetGroup(address)); return GetDeterministicRandomizer(RANDOMIZER_ID_NETGROUP).Write(vchNetGroup.data(), vchNetGroup.size()).Finalize(); } @@ -3135,7 +3153,7 @@ void CaptureMessageToFile(const CAddress& addr, std::string clean_addr = addr.ToString(); std::replace(clean_addr.begin(), clean_addr.end(), ':', '_'); - fs::path base_path = gArgs.GetDataDirNet() / "message_capture" / clean_addr; + fs::path base_path = gArgs.GetDataDirNet() / "message_capture" / fs::u8path(clean_addr); fs::create_directories(base_path); fs::path path = base_path / (is_incoming ? "msgs_recv.dat" : "msgs_sent.dat"); @@ -13,9 +13,11 @@ #include <crypto/siphash.h> #include <hash.h> #include <i2p.h> +#include <logging.h> #include <net_permissions.h> #include <netaddress.h> #include <netbase.h> +#include <netgroup.h> #include <policy/feerate.h> #include <protocol.h> #include <random.h> @@ -32,6 +34,7 @@ #include <cstdint> #include <deque> #include <functional> +#include <list> #include <map> #include <memory> #include <optional> @@ -99,15 +102,22 @@ struct AddedNodeInfo class CNodeStats; class CClientUIInterface; -struct CSerializedNetMsg -{ +struct CSerializedNetMsg { CSerializedNetMsg() = default; CSerializedNetMsg(CSerializedNetMsg&&) = default; CSerializedNetMsg& operator=(CSerializedNetMsg&&) = default; - // No copying, only moves. + // No implicit copying, only moves. CSerializedNetMsg(const CSerializedNetMsg& msg) = delete; CSerializedNetMsg& operator=(const CSerializedNetMsg&) = delete; + CSerializedNetMsg Copy() const + { + CSerializedNetMsg copy; + copy.data = data; + copy.m_type = m_type; + return copy; + } + std::vector<unsigned char> data; std::string m_type; }; @@ -242,15 +252,14 @@ struct LocalServiceInfo { extern Mutex g_maplocalhost_mutex; extern std::map<CNetAddr, LocalServiceInfo> mapLocalHost GUARDED_BY(g_maplocalhost_mutex); -extern const std::string NET_MESSAGE_COMMAND_OTHER; -typedef std::map<std::string, uint64_t> mapMsgCmdSize; //command, total bytes +extern const std::string NET_MESSAGE_TYPE_OTHER; +using mapMsgTypeSize = std::map</* message type */ std::string, /* total bytes */ uint64_t>; class CNodeStats { public: NodeId nodeid; ServiceFlags nServices; - bool fRelayTxes; std::chrono::seconds m_last_send; std::chrono::seconds m_last_recv; std::chrono::seconds m_last_tx_time; @@ -265,13 +274,12 @@ public: bool m_bip152_highbandwidth_from; int m_starting_height; uint64_t nSendBytes; - mapMsgCmdSize mapSendBytesPerMsgCmd; + mapMsgTypeSize mapSendBytesPerMsgType; uint64_t nRecvBytes; - mapMsgCmdSize mapRecvBytesPerMsgCmd; + mapMsgTypeSize mapRecvBytesPerMsgType; NetPermissionFlags m_permissionFlags; std::chrono::microseconds m_last_ping_time; std::chrono::microseconds m_min_ping_time; - CAmount minFeeFilter; // Our address, as reported by the peer std::string addrLocal; // Address of this peer @@ -307,7 +315,7 @@ public: /** The TransportDeserializer takes care of holding and deserializing the * network receive buffer. It can deserialize the network buffer into a - * transport protocol agnostic CNetMessage (command & payload) + * transport protocol agnostic CNetMessage (message type & payload) */ class TransportDeserializer { public: @@ -548,34 +556,15 @@ public: // Peer selected us as (compact blocks) high-bandwidth peer (BIP152) std::atomic<bool> m_bip152_highbandwidth_from{false}; - struct TxRelay { - mutable RecursiveMutex cs_filter; - // We use fRelayTxes for two purposes - - // a) it allows us to not relay tx invs before receiving the peer's version message - // b) the peer may tell us in its version message that we should not relay tx invs - // unless it loads a bloom filter. - bool fRelayTxes GUARDED_BY(cs_filter){false}; - std::unique_ptr<CBloomFilter> pfilter PT_GUARDED_BY(cs_filter) GUARDED_BY(cs_filter){nullptr}; - - mutable RecursiveMutex cs_tx_inventory; - CRollingBloomFilter filterInventoryKnown GUARDED_BY(cs_tx_inventory){50000, 0.000001}; - // Set of transaction ids we still have to announce. - // They are sorted by the mempool before relay, so the order is not important. - std::set<uint256> setInventoryTxToSend; - // Used for BIP35 mempool sending - bool fSendMempool GUARDED_BY(cs_tx_inventory){false}; - // Last time a "MEMPOOL" request was serviced. - std::atomic<std::chrono::seconds> m_last_mempool_req{0s}; - std::chrono::microseconds nNextInvSend{0}; - - /** Minimum fee rate with which to filter inv's to this node */ - std::atomic<CAmount> minFeeFilter{0}; - CAmount lastSentFeeFilter{0}; - std::chrono::microseconds m_next_send_feefilter{0}; - }; + /** Whether we should relay transactions to this peer (their version + * message did not include fRelay=false and this is not a block-relay-only + * connection). This only changes from false to true. It will never change + * back to false. Used only in inbound eviction logic. */ + std::atomic_bool m_relays_txs{false}; - // m_tx_relay == nullptr if we're not relaying transactions with this peer - std::unique_ptr<TxRelay> m_tx_relay; + /** Whether this peer has loaded a bloom filter. Used only in inbound + * eviction logic. */ + std::atomic_bool m_bloom_filter_loaded{false}; /** UNIX epoch time of the last block received from this peer that we had * not yet seen (e.g. not already received from another peer), that passed @@ -651,23 +640,6 @@ public: nRefCount--; } - void AddKnownTx(const uint256& hash) - { - if (m_tx_relay != nullptr) { - LOCK(m_tx_relay->cs_tx_inventory); - m_tx_relay->filterInventoryKnown.insert(hash); - } - } - - void PushTxInventory(const uint256& hash) - { - if (m_tx_relay == nullptr) return; - LOCK(m_tx_relay->cs_tx_inventory); - if (!m_tx_relay->filterInventoryKnown.contains(hash)) { - m_tx_relay->setInventoryTxToSend.insert(hash); - } - } - void CloseSocketDisconnect(); void CopyStats(CNodeStats& stats); @@ -714,8 +686,8 @@ private: CService addrLocal GUARDED_BY(m_addr_local_mutex); mutable Mutex m_addr_local_mutex; - mapMsgCmdSize mapSendBytesPerMsgCmd GUARDED_BY(cs_vSend); - mapMsgCmdSize mapRecvBytesPerMsgCmd GUARDED_BY(cs_vRecv); + mapMsgTypeSize mapSendBytesPerMsgType GUARDED_BY(cs_vSend); + mapMsgTypeSize mapRecvBytesPerMsgType GUARDED_BY(cs_vRecv); }; /** @@ -789,7 +761,10 @@ public: bool m_i2p_accept_incoming; }; - void Init(const Options& connOptions) { + void Init(const Options& connOptions) EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex) + { + AssertLockNotHeld(m_total_bytes_sent_mutex); + nLocalServices = connOptions.nLocalServices; nMaxConnections = connOptions.nMaxConnections; m_max_outbound_full_relay = std::min(connOptions.m_max_outbound_full_relay, connOptions.nMaxConnections); @@ -805,7 +780,7 @@ public: nReceiveFloodSize = connOptions.nReceiveFloodSize; m_peer_connect_timeout = std::chrono::seconds{connOptions.m_peer_connect_timeout}; { - LOCK(cs_totalBytesSent); + LOCK(m_total_bytes_sent_mutex); nMaxOutboundLimit = connOptions.nMaxOutboundLimit; } vWhitelistedRange = connOptions.vWhitelistedRange; @@ -816,9 +791,11 @@ public: m_onion_binds = connOptions.onion_binds; } - CConnman(uint64_t seed0, uint64_t seed1, AddrMan& addrman, bool network_active = true); + CConnman(uint64_t seed0, uint64_t seed1, AddrMan& addrman, const NetGroupManager& netgroupman, + bool network_active = true); + ~CConnman(); - bool Start(CScheduler& scheduler, const Options& options); + bool Start(CScheduler& scheduler, const Options& options) EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); void StopThreads(); void StopNodes(); @@ -837,7 +814,7 @@ public: bool ForNode(NodeId id, std::function<bool(CNode* pnode)> func); - void PushMessage(CNode* pnode, CSerializedNetMsg&& msg); + void PushMessage(CNode* pnode, CSerializedNetMsg&& msg) EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); using NodeFn = std::function<void(CNode*)>; void ForEachNode(const NodeFn& func) @@ -928,24 +905,22 @@ public: //! that peer during `net_processing.cpp:PushNodeVersion()`. ServiceFlags GetLocalServices() const; - uint64_t GetMaxOutboundTarget() const; + uint64_t GetMaxOutboundTarget() const EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); std::chrono::seconds GetMaxOutboundTimeframe() const; //! check if the outbound target is reached //! if param historicalBlockServingLimit is set true, the function will //! response true if the limit for serving historical blocks has been reached - bool OutboundTargetReached(bool historicalBlockServingLimit) const; + bool OutboundTargetReached(bool historicalBlockServingLimit) const EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); //! response the bytes left in the current max outbound cycle //! in case of no limit, it will always response 0 - uint64_t GetOutboundTargetBytesLeft() const; + uint64_t GetOutboundTargetBytesLeft() const EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); - //! returns the time left in the current max outbound cycle - //! in case of no limit, it will always return 0 - std::chrono::seconds GetMaxOutboundTimeLeftInCycle() const; + std::chrono::seconds GetMaxOutboundTimeLeftInCycle() const EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); uint64_t GetTotalBytesRecv() const; - uint64_t GetTotalBytesSent() const; + uint64_t GetTotalBytesSent() const EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); /** Get a unique deterministic randomizer. */ CSipHasher GetDeterministicRandomizer(uint64_t id) const; @@ -971,6 +946,10 @@ private: NetPermissionFlags m_permissions; }; + //! returns the time left in the current max outbound cycle + //! in case of no limit, it will always return 0 + std::chrono::seconds GetMaxOutboundTimeLeftInCycle_() const EXCLUSIVE_LOCKS_REQUIRED(m_total_bytes_sent_mutex); + bool BindListenPort(const CService& bindAddr, bilingual_str& strError, NetPermissionFlags permissions); bool Bind(const CService& addr, unsigned int flags, NetPermissionFlags permissions); bool InitBinds(const Options& options); @@ -1030,7 +1009,7 @@ private: /** * Check connected and listening sockets for IO readiness and process them accordingly. */ - void SocketHandler(); + void SocketHandler() EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); /** * Do the read/write for connected sockets that are ready for IO. @@ -1043,7 +1022,8 @@ private: void SocketHandlerConnected(const std::vector<CNode*>& nodes, const std::set<SOCKET>& recv_set, const std::set<SOCKET>& send_set, - const std::set<SOCKET>& error_set); + const std::set<SOCKET>& error_set) + EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); /** * Accept incoming connections, one from each read-ready listening socket. @@ -1051,7 +1031,7 @@ private: */ void SocketHandlerListening(const std::set<SOCKET>& recv_set); - void ThreadSocketHandler(); + void ThreadSocketHandler() EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); void ThreadDNSAddressSeed(); uint64_t CalculateKeyedNetGroup(const CAddress& ad) const; @@ -1080,7 +1060,7 @@ private: // Network stats void RecordBytesRecv(uint64_t bytes); - void RecordBytesSent(uint64_t bytes); + void RecordBytesSent(uint64_t bytes) EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); /** * Return vector of current BLOCK_RELAY peers. @@ -1091,14 +1071,14 @@ private: static bool NodeFullyConnected(const CNode* pnode); // Network usage totals - mutable RecursiveMutex cs_totalBytesSent; + mutable Mutex m_total_bytes_sent_mutex; std::atomic<uint64_t> nTotalBytesRecv{0}; - uint64_t nTotalBytesSent GUARDED_BY(cs_totalBytesSent) {0}; + uint64_t nTotalBytesSent GUARDED_BY(m_total_bytes_sent_mutex) {0}; // outbound limit & stats - uint64_t nMaxOutboundTotalBytesSentInCycle GUARDED_BY(cs_totalBytesSent) {0}; - std::chrono::seconds nMaxOutboundCycleStartTime GUARDED_BY(cs_totalBytesSent) {0}; - uint64_t nMaxOutboundLimit GUARDED_BY(cs_totalBytesSent); + uint64_t nMaxOutboundTotalBytesSentInCycle GUARDED_BY(m_total_bytes_sent_mutex) {0}; + std::chrono::seconds nMaxOutboundCycleStartTime GUARDED_BY(m_total_bytes_sent_mutex) {0}; + uint64_t nMaxOutboundLimit GUARDED_BY(m_total_bytes_sent_mutex); // P2P timeout in seconds std::chrono::seconds m_peer_connect_timeout; @@ -1114,6 +1094,7 @@ private: std::atomic<bool> fNetworkActive{true}; bool fAddressesInitialized{false}; AddrMan& addrman; + const NetGroupManager& m_netgroupman; std::deque<std::string> m_addr_fetches GUARDED_BY(m_addr_fetches_mutex); Mutex m_addr_fetches_mutex; std::vector<std::string> m_added_nodes GUARDED_BY(m_added_nodes_mutex); @@ -1301,7 +1282,7 @@ struct NodeEvictionCandidate std::chrono::seconds m_last_block_time; std::chrono::seconds m_last_tx_time; bool fRelevantServices; - bool fRelayTxes; + bool m_relay_txs; bool fBloomFilter; uint64_t nKeyedNetGroup; bool prefer_evict; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index aee1e4c30c..478368b673 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -41,6 +41,7 @@ #include <algorithm> #include <atomic> #include <chrono> +#include <future> #include <memory> #include <optional> #include <typeinfo> @@ -134,6 +135,8 @@ static const unsigned int NODE_NETWORK_LIMITED_MIN_BLOCKS = 288; static constexpr auto AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL{24h}; /** Average delay between peer address broadcasts */ static constexpr auto AVG_ADDRESS_BROADCAST_INTERVAL{30s}; +/** Delay between rotating the peers we relay a particular address to */ +static constexpr auto ROTATE_ADDR_RELAY_DEST_INTERVAL{24h}; /** Average delay between trickled inventory transmissions for inbound peers. * Blocks and peers with NetPermissionFlags::NoBan permission bypass this. */ static constexpr auto INBOUND_INVENTORY_BROADCAST_INTERVAL{5s}; @@ -232,6 +235,39 @@ struct Peer { /** Whether a ping has been requested by the user */ std::atomic<bool> m_ping_queued{false}; + /** Whether this peer relays txs via wtxid */ + std::atomic<bool> m_wtxid_relay{false}; + + struct TxRelay { + mutable RecursiveMutex m_bloom_filter_mutex; + // We use m_relay_txs for two purposes - + // a) it allows us to not relay tx invs before receiving the peer's version message + // b) the peer may tell us in its version message that we should not relay tx invs + // unless it loads a bloom filter. + bool m_relay_txs GUARDED_BY(m_bloom_filter_mutex){false}; + std::unique_ptr<CBloomFilter> m_bloom_filter PT_GUARDED_BY(m_bloom_filter_mutex) GUARDED_BY(m_bloom_filter_mutex){nullptr}; + + mutable RecursiveMutex m_tx_inventory_mutex; + CRollingBloomFilter m_tx_inventory_known_filter GUARDED_BY(m_tx_inventory_mutex){50000, 0.000001}; + // Set of transaction ids we still have to announce. + // They are sorted by the mempool before relay, so the order is not important. + std::set<uint256> m_tx_inventory_to_send; + // Used for BIP35 mempool sending + bool m_send_mempool GUARDED_BY(m_tx_inventory_mutex){false}; + // Last time a "MEMPOOL" request was serviced. + std::atomic<std::chrono::seconds> m_last_mempool_req{0s}; + std::chrono::microseconds m_next_inv_send_time{0}; + + /** Minimum fee rate with which to filter inv's to this node */ + std::atomic<CAmount> m_fee_filter_received{0}; + CAmount m_fee_filter_sent{0}; + std::chrono::microseconds m_next_send_feefilter{0}; + }; + + /** Transaction relay data. Will be a nullptr if we're not relaying + * transactions with this peer (e.g. if it's a block-relay-only peer) */ + std::unique_ptr<TxRelay> m_tx_relay; + /** A vector of addresses to send to the peer, limited to MAX_ADDR_TO_SEND. */ std::vector<CAddress> m_addrs_to_send; /** Probabilistic filter to track recent addr messages relayed with this @@ -290,13 +326,112 @@ struct Peer { /** Work queue of items requested by this peer **/ std::deque<CInv> m_getdata_requests GUARDED_BY(m_getdata_requests_mutex); - explicit Peer(NodeId id) + explicit Peer(NodeId id, bool tx_relay) : m_id(id) + , m_tx_relay(tx_relay ? std::make_unique<TxRelay>() : nullptr) {} }; using PeerRef = std::shared_ptr<Peer>; +/** + * Maintain validation-specific state about nodes, protected by cs_main, instead + * by CNode's own locks. This simplifies asynchronous operation, where + * processing of incoming data is done after the ProcessMessage call returns, + * and we're no longer holding the node's locks. + */ +struct CNodeState { + //! The best known block we know this peer has announced. + const CBlockIndex* pindexBestKnownBlock{nullptr}; + //! The hash of the last unknown block this peer has announced. + uint256 hashLastUnknownBlock{}; + //! The last full block we both have. + const CBlockIndex* pindexLastCommonBlock{nullptr}; + //! The best header we have sent our peer. + const CBlockIndex* pindexBestHeaderSent{nullptr}; + //! Length of current-streak of unconnecting headers announcements + int nUnconnectingHeaders{0}; + //! Whether we've started headers synchronization with this peer. + bool fSyncStarted{false}; + //! When to potentially disconnect peer for stalling headers download + std::chrono::microseconds m_headers_sync_timeout{0us}; + //! Since when we're stalling block download progress (in microseconds), or 0. + std::chrono::microseconds m_stalling_since{0us}; + std::list<QueuedBlock> vBlocksInFlight; + //! When the first entry in vBlocksInFlight started downloading. Don't care when vBlocksInFlight is empty. + std::chrono::microseconds m_downloading_since{0us}; + int nBlocksInFlight{0}; + //! Whether we consider this a preferred download peer. + bool fPreferredDownload{false}; + //! Whether this peer wants invs or headers (when possible) for block announcements. + bool fPreferHeaders{false}; + //! Whether this peer wants invs or cmpctblocks (when possible) for block announcements. + bool fPreferHeaderAndIDs{false}; + /** + * Whether this peer will send us cmpctblocks if we request them. + * This is not used to gate request logic, as we really only care about fSupportsDesiredCmpctVersion, + * but is used as a flag to "lock in" the version of compact blocks (fWantsCmpctWitness) we send. + */ + bool fProvidesHeaderAndIDs{false}; + //! Whether this peer can give us witnesses + bool fHaveWitness{false}; + //! Whether this peer wants witnesses in cmpctblocks/blocktxns + bool fWantsCmpctWitness{false}; + /** + * If we've announced NODE_WITNESS to this peer: whether the peer sends witnesses in cmpctblocks/blocktxns, + * otherwise: whether this peer sends non-witnesses in cmpctblocks/blocktxns. + */ + bool fSupportsDesiredCmpctVersion{false}; + + /** State used to enforce CHAIN_SYNC_TIMEOUT and EXTRA_PEER_CHECK_INTERVAL logic. + * + * Both are only in effect for outbound, non-manual, non-protected connections. + * Any peer protected (m_protect = true) is not chosen for eviction. A peer is + * marked as protected if all of these are true: + * - its connection type is IsBlockOnlyConn() == false + * - it gave us a valid connecting header + * - we haven't reached MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT yet + * - its chain tip has at least as much work as ours + * + * CHAIN_SYNC_TIMEOUT: if a peer's best known block has less work than our tip, + * set a timeout CHAIN_SYNC_TIMEOUT in the future: + * - If at timeout their best known block now has more work than our tip + * when the timeout was set, then either reset the timeout or clear it + * (after comparing against our current tip's work) + * - If at timeout their best known block still has less work than our + * tip did when the timeout was set, then send a getheaders message, + * and set a shorter timeout, HEADERS_RESPONSE_TIME seconds in future. + * If their best known block is still behind when that new timeout is + * reached, disconnect. + * + * EXTRA_PEER_CHECK_INTERVAL: after each interval, if we have too many outbound peers, + * drop the outbound one that least recently announced us a new block. + */ + struct ChainSyncTimeoutState { + //! A timeout used for checking whether our peer has sufficiently synced + std::chrono::seconds m_timeout{0s}; + //! A header with the work we require on our peer's chain + const CBlockIndex* m_work_header{nullptr}; + //! After timeout is reached, set to true after sending getheaders + bool m_sent_getheaders{false}; + //! Whether this peer is protected from disconnection due to a bad/slow chain + bool m_protect{false}; + }; + + ChainSyncTimeoutState m_chain_sync; + + //! Time of last new block announcement + int64_t m_last_block_announcement{0}; + + //! Whether this peer is an inbound connection + const bool m_is_inbound; + + //! A rolling bloom filter of all announced tx CInvs to this peer. + CRollingBloomFilter m_recently_announced_invs = CRollingBloomFilter{INVENTORY_MAX_RECENT_RELAY, 0.000001}; + + CNodeState(bool is_inbound) : m_is_inbound(is_inbound) {} +}; + class PeerManagerImpl final : public PeerManager { public: @@ -329,11 +464,9 @@ public: void Misbehaving(const NodeId pnode, const int howmuch, const std::string& message) override; void ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, const std::chrono::microseconds time_received, const std::atomic<bool>& interruptMsgProc) override; + void UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_seconds) override; private: - void _RelayTransaction(const uint256& txid, const uint256& wtxid) - EXCLUSIVE_LOCKS_REQUIRED(cs_main); - /** Consider evicting an outbound peer based on the amount of time they've been behind our tip */ void ConsiderEviction(CNode& pto, std::chrono::seconds time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -394,7 +527,7 @@ private: EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** Send a version message to a peer */ - void PushNodeVersion(CNode& pnode); + void PushNodeVersion(CNode& pnode, const Peer& peer); /** Send a ping message every PING_INTERVAL or if requested via RPC. May * mark the peer to be disconnected if a ping has timed out. @@ -415,7 +548,7 @@ private: void RelayAddress(NodeId originator, const CAddress& addr, bool fReachable); /** Send `feefilter` message. */ - void MaybeSendFeefilter(CNode& node, std::chrono::microseconds current_time) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void MaybeSendFeefilter(CNode& node, Peer& peer, std::chrono::microseconds current_time); const CChainParams& m_chainparams; CConnman& m_connman; @@ -450,6 +583,16 @@ private: */ std::map<NodeId, PeerRef> m_peer_map GUARDED_BY(m_peer_mutex); + /** Map maintaining per-node state. */ + std::map<NodeId, CNodeState> m_node_states GUARDED_BY(cs_main); + + /** Get a pointer to a const CNodeState, used when not mutating the CNodeState object. */ + const CNodeState* State(NodeId pnode) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); + /** Get a pointer to a mutable CNodeState. */ + CNodeState* State(NodeId pnode) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + uint32_t GetFetchFlags(const CNode& pfrom) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); + std::atomic<std::chrono::microseconds> m_next_inv_to_inbounds{0us}; /** Number of nodes with fSyncStarted. */ @@ -464,11 +607,14 @@ private: std::map<uint256, std::pair<NodeId, bool>> mapBlockSource GUARDED_BY(cs_main); /** Number of peers with wtxid relay. */ - int m_wtxid_relay_peers GUARDED_BY(cs_main) = 0; + std::atomic<int> m_wtxid_relay_peers{0}; /** Number of outbound peers with m_chain_sync.m_protect. */ int m_outbound_peers_with_protect_from_disconnect GUARDED_BY(cs_main) = 0; + /** Number of preferable block download peers. */ + int m_num_preferred_download_peers GUARDED_BY(cs_main){0}; + bool AlreadyHaveTx(const GenTxid& gtxid) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** @@ -535,6 +681,17 @@ private: std::chrono::microseconds NextInvToInbounds(std::chrono::microseconds now, std::chrono::seconds average_interval); + + // All of the following cache a recent block, and are protected by m_most_recent_block_mutex + RecursiveMutex m_most_recent_block_mutex; + std::shared_ptr<const CBlock> m_most_recent_block GUARDED_BY(m_most_recent_block_mutex); + std::shared_ptr<const CBlockHeaderAndShortTxIDs> m_most_recent_compact_block GUARDED_BY(m_most_recent_block_mutex); + uint256 m_most_recent_block_hash GUARDED_BY(m_most_recent_block_mutex); + bool m_most_recent_compact_block_has_witnesses GUARDED_BY(m_most_recent_block_mutex){false}; + + /** Height of the highest block announced using BIP 152 high-bandwidth mode. */ + int m_highest_fast_announce{0}; + /** Have we requested this block from a peer */ bool IsBlockRequested(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -676,125 +833,20 @@ private: */ bool SetupAddressRelay(const CNode& node, Peer& peer); }; -} // namespace - -namespace { - /** Number of preferable block download peers. */ - int nPreferredDownload GUARDED_BY(cs_main) = 0; -} // namespace - -namespace { -/** - * Maintain validation-specific state about nodes, protected by cs_main, instead - * by CNode's own locks. This simplifies asynchronous operation, where - * processing of incoming data is done after the ProcessMessage call returns, - * and we're no longer holding the node's locks. - */ -struct CNodeState { - //! The best known block we know this peer has announced. - const CBlockIndex* pindexBestKnownBlock{nullptr}; - //! The hash of the last unknown block this peer has announced. - uint256 hashLastUnknownBlock{}; - //! The last full block we both have. - const CBlockIndex* pindexLastCommonBlock{nullptr}; - //! The best header we have sent our peer. - const CBlockIndex* pindexBestHeaderSent{nullptr}; - //! Length of current-streak of unconnecting headers announcements - int nUnconnectingHeaders{0}; - //! Whether we've started headers synchronization with this peer. - bool fSyncStarted{false}; - //! When to potentially disconnect peer for stalling headers download - std::chrono::microseconds m_headers_sync_timeout{0us}; - //! Since when we're stalling block download progress (in microseconds), or 0. - std::chrono::microseconds m_stalling_since{0us}; - std::list<QueuedBlock> vBlocksInFlight; - //! When the first entry in vBlocksInFlight started downloading. Don't care when vBlocksInFlight is empty. - std::chrono::microseconds m_downloading_since{0us}; - int nBlocksInFlight{0}; - //! Whether we consider this a preferred download peer. - bool fPreferredDownload{false}; - //! Whether this peer wants invs or headers (when possible) for block announcements. - bool fPreferHeaders{false}; - //! Whether this peer wants invs or cmpctblocks (when possible) for block announcements. - bool fPreferHeaderAndIDs{false}; - /** - * Whether this peer will send us cmpctblocks if we request them. - * This is not used to gate request logic, as we really only care about fSupportsDesiredCmpctVersion, - * but is used as a flag to "lock in" the version of compact blocks (fWantsCmpctWitness) we send. - */ - bool fProvidesHeaderAndIDs{false}; - //! Whether this peer can give us witnesses - bool fHaveWitness{false}; - //! Whether this peer wants witnesses in cmpctblocks/blocktxns - bool fWantsCmpctWitness{false}; - /** - * If we've announced NODE_WITNESS to this peer: whether the peer sends witnesses in cmpctblocks/blocktxns, - * otherwise: whether this peer sends non-witnesses in cmpctblocks/blocktxns. - */ - bool fSupportsDesiredCmpctVersion{false}; - - /** State used to enforce CHAIN_SYNC_TIMEOUT and EXTRA_PEER_CHECK_INTERVAL logic. - * - * Both are only in effect for outbound, non-manual, non-protected connections. - * Any peer protected (m_protect = true) is not chosen for eviction. A peer is - * marked as protected if all of these are true: - * - its connection type is IsBlockOnlyConn() == false - * - it gave us a valid connecting header - * - we haven't reached MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT yet - * - its chain tip has at least as much work as ours - * - * CHAIN_SYNC_TIMEOUT: if a peer's best known block has less work than our tip, - * set a timeout CHAIN_SYNC_TIMEOUT in the future: - * - If at timeout their best known block now has more work than our tip - * when the timeout was set, then either reset the timeout or clear it - * (after comparing against our current tip's work) - * - If at timeout their best known block still has less work than our - * tip did when the timeout was set, then send a getheaders message, - * and set a shorter timeout, HEADERS_RESPONSE_TIME seconds in future. - * If their best known block is still behind when that new timeout is - * reached, disconnect. - * - * EXTRA_PEER_CHECK_INTERVAL: after each interval, if we have too many outbound peers, - * drop the outbound one that least recently announced us a new block. - */ - struct ChainSyncTimeoutState { - //! A timeout used for checking whether our peer has sufficiently synced - std::chrono::seconds m_timeout{0s}; - //! A header with the work we require on our peer's chain - const CBlockIndex* m_work_header{nullptr}; - //! After timeout is reached, set to true after sending getheaders - bool m_sent_getheaders{false}; - //! Whether this peer is protected from disconnection due to a bad/slow chain - bool m_protect{false}; - }; - - ChainSyncTimeoutState m_chain_sync; - - //! Time of last new block announcement - int64_t m_last_block_announcement{0}; - - //! Whether this peer is an inbound connection - const bool m_is_inbound; - - //! A rolling bloom filter of all announced tx CInvs to this peer. - CRollingBloomFilter m_recently_announced_invs = CRollingBloomFilter{INVENTORY_MAX_RECENT_RELAY, 0.000001}; - - //! Whether this peer relays txs via wtxid - bool m_wtxid_relay{false}; - - CNodeState(bool is_inbound) : m_is_inbound(is_inbound) {} -}; - -/** Map maintaining per-node state. */ -static std::map<NodeId, CNodeState> mapNodeState GUARDED_BY(cs_main); -static CNodeState *State(NodeId pnode) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - std::map<NodeId, CNodeState>::iterator it = mapNodeState.find(pnode); - if (it == mapNodeState.end()) +const CNodeState* PeerManagerImpl::State(NodeId pnode) const EXCLUSIVE_LOCKS_REQUIRED(cs_main) +{ + std::map<NodeId, CNodeState>::const_iterator it = m_node_states.find(pnode); + if (it == m_node_states.end()) return nullptr; return &it->second; } +CNodeState* PeerManagerImpl::State(NodeId pnode) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +{ + return const_cast<CNodeState*>(std::as_const(*this).State(pnode)); +} + /** * Whether the peer supports the address. For example, a peer that does not * implement BIP155 cannot receive Tor v3 addresses because it requires @@ -826,14 +878,12 @@ static void PushAddress(Peer& peer, const CAddress& addr, FastRandomContext& ins } } -static void UpdatePreferredDownload(const CNode& node, CNodeState* state) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +static void AddKnownTx(Peer& peer, const uint256& hash) { - nPreferredDownload -= state->fPreferredDownload; - - // Whether this node should be marked as a preferred download node. - state->fPreferredDownload = (!node.IsInboundConn() || node.HasPermission(NetPermissionFlags::NoBan)) && !node.IsAddrFetchConn() && !node.fClient; - - nPreferredDownload += state->fPreferredDownload; + if (peer.m_tx_relay != nullptr) { + LOCK(peer.m_tx_relay->m_tx_inventory_mutex); + peer.m_tx_relay->m_tx_inventory_known_filter.insert(hash); + } } std::chrono::microseconds PeerManagerImpl::NextInvToInbounds(std::chrono::microseconds now, @@ -1121,7 +1171,7 @@ void PeerManagerImpl::FindNextBlocksToDownload(NodeId nodeid, unsigned int count } // namespace -void PeerManagerImpl::PushNodeVersion(CNode& pnode) +void PeerManagerImpl::PushNodeVersion(CNode& pnode, const Peer& peer) { // Note that pnode->GetLocalServices() is a reflection of the local // services we were offering when the CNode object was created for this @@ -1136,7 +1186,7 @@ void PeerManagerImpl::PushNodeVersion(CNode& pnode) CService addr_you = addr.IsRoutable() && !IsProxy(addr) && addr.IsAddrV1Compatible() ? addr : CService(); uint64_t your_services{addr.nServices}; - const bool tx_relay = !m_ignore_incoming_txs && pnode.m_tx_relay != nullptr && !pnode.IsFeelerConn(); + const bool tx_relay = !m_ignore_incoming_txs && peer.m_tx_relay != nullptr && !pnode.IsFeelerConn(); m_connman.PushMessage(&pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, my_services, nTime, your_services, addr_you, // Together the pre-version-31402 serialization of CAddress "addrYou" (without nTime) my_services, CService(), // Together the pre-version-31402 serialization of CAddress "addrMe" (without nTime) @@ -1176,9 +1226,7 @@ void PeerManagerImpl::AddTxAnnouncement(const CNode& node, const GenTxid& gtxid, m_txrequest.ReceivedInv(nodeid, gtxid, preferred, current_time + delay); } -// This function is used for testing the stale tip eviction logic, see -// denialofservice_tests.cpp -void UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_seconds) +void PeerManagerImpl::UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_seconds) { LOCK(cs_main); CNodeState *state = State(node); @@ -1190,16 +1238,16 @@ void PeerManagerImpl::InitializeNode(CNode *pnode) NodeId nodeid = pnode->GetId(); { LOCK(cs_main); - mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(pnode->IsInboundConn())); + m_node_states.emplace_hint(m_node_states.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(pnode->IsInboundConn())); assert(m_txrequest.Count(nodeid) == 0); } + PeerRef peer = std::make_shared<Peer>(nodeid, /*tx_relay=*/ !pnode->IsBlockOnlyConn()); { - PeerRef peer = std::make_shared<Peer>(nodeid); LOCK(m_peer_mutex); - m_peer_map.emplace_hint(m_peer_map.end(), nodeid, std::move(peer)); + m_peer_map.emplace_hint(m_peer_map.end(), nodeid, peer); } if (!pnode->IsInboundConn()) { - PushNodeVersion(*pnode); + PushNodeVersion(*pnode, *peer); } } @@ -1211,8 +1259,7 @@ void PeerManagerImpl::ReattemptInitialBroadcast(CScheduler& scheduler) CTransactionRef tx = m_mempool.get(txid); if (tx != nullptr) { - LOCK(cs_main); - _RelayTransaction(txid, tx->GetWitnessHash()); + RelayTransaction(txid, tx->GetWitnessHash()); } else { m_mempool.RemoveUnbroadcastTx(txid, true); } @@ -1239,6 +1286,8 @@ void PeerManagerImpl::FinalizeNode(const CNode& node) PeerRef peer = RemovePeer(nodeid); assert(peer != nullptr); misbehavior = WITH_LOCK(peer->m_misbehavior_mutex, return peer->m_misbehavior_score); + m_wtxid_relay_peers -= peer->m_wtxid_relay; + assert(m_wtxid_relay_peers >= 0); } CNodeState *state = State(nodeid); assert(state != nullptr); @@ -1251,20 +1300,18 @@ void PeerManagerImpl::FinalizeNode(const CNode& node) } WITH_LOCK(g_cs_orphans, m_orphanage.EraseForPeer(nodeid)); m_txrequest.DisconnectedPeer(nodeid); - nPreferredDownload -= state->fPreferredDownload; + m_num_preferred_download_peers -= state->fPreferredDownload; m_peers_downloading_from -= (state->nBlocksInFlight != 0); assert(m_peers_downloading_from >= 0); m_outbound_peers_with_protect_from_disconnect -= state->m_chain_sync.m_protect; assert(m_outbound_peers_with_protect_from_disconnect >= 0); - m_wtxid_relay_peers -= state->m_wtxid_relay; - assert(m_wtxid_relay_peers >= 0); - mapNodeState.erase(nodeid); + m_node_states.erase(nodeid); - if (mapNodeState.empty()) { + if (m_node_states.empty()) { // Do a consistency check after the last peer is removed. assert(mapBlocksInFlight.empty()); - assert(nPreferredDownload == 0); + assert(m_num_preferred_download_peers == 0); assert(m_peers_downloading_from == 0); assert(m_outbound_peers_with_protect_from_disconnect == 0); assert(m_wtxid_relay_peers == 0); @@ -1305,7 +1352,7 @@ bool PeerManagerImpl::GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) c { { LOCK(cs_main); - CNodeState* state = State(nodeid); + const CNodeState* state = State(nodeid); if (state == nullptr) return false; stats.nSyncHeight = state->pindexBestKnownBlock ? state->pindexBestKnownBlock->nHeight : -1; @@ -1330,6 +1377,14 @@ bool PeerManagerImpl::GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) c ping_wait = GetTime<std::chrono::microseconds>() - peer->m_ping_start.load(); } + if (peer->m_tx_relay != nullptr) { + stats.m_relay_txs = WITH_LOCK(peer->m_tx_relay->m_bloom_filter_mutex, return peer->m_tx_relay->m_relay_txs); + stats.m_fee_filter_received = peer->m_tx_relay->m_fee_filter_received.load(); + } else { + stats.m_relay_txs = false; + stats.m_fee_filter_received = 0; + } + stats.m_ping_wait = ping_wait; stats.m_addr_processed = peer->m_addr_processed.load(); stats.m_addr_rate_limited = peer->m_addr_rate_limited.load(); @@ -1455,9 +1510,9 @@ bool PeerManagerImpl::BlockRequestAllowed(const CBlockIndex* pindex) { AssertLockHeld(cs_main); if (m_chainman.ActiveChain().Contains(pindex)) return true; - return pindex->IsValid(BLOCK_VALID_SCRIPTS) && (pindexBestHeader != nullptr) && - (pindexBestHeader->GetBlockTime() - pindex->GetBlockTime() < STALE_RELAY_AGE_LIMIT) && - (GetBlockProofEquivalentTime(*pindexBestHeader, *pindex, *pindexBestHeader, m_chainparams.GetConsensus()) < STALE_RELAY_AGE_LIMIT); + return pindex->IsValid(BLOCK_VALID_SCRIPTS) && (m_chainman.m_best_header != nullptr) && + (m_chainman.m_best_header->GetBlockTime() - pindex->GetBlockTime() < STALE_RELAY_AGE_LIMIT) && + (GetBlockProofEquivalentTime(*m_chainman.m_best_header, *pindex, *m_chainman.m_best_header, m_chainparams.GetConsensus()) < STALE_RELAY_AGE_LIMIT); } std::optional<std::string> PeerManagerImpl::FetchBlock(NodeId peer_id, const CBlockIndex& block_index) @@ -1571,13 +1626,6 @@ void PeerManagerImpl::BlockDisconnected(const std::shared_ptr<const CBlock> &blo m_recent_confirmed_transactions.reset(); } -// All of the following cache a recent block, and are protected by cs_most_recent_block -static RecursiveMutex cs_most_recent_block; -static std::shared_ptr<const CBlock> most_recent_block GUARDED_BY(cs_most_recent_block); -static std::shared_ptr<const CBlockHeaderAndShortTxIDs> most_recent_compact_block GUARDED_BY(cs_most_recent_block); -static uint256 most_recent_block_hash GUARDED_BY(cs_most_recent_block); -static bool fWitnessesPresentInMostRecentCompactBlock GUARDED_BY(cs_most_recent_block); - /** * Maintain state about the best-seen block and fast-announce a compact block * to compatible peers. @@ -1589,26 +1637,26 @@ void PeerManagerImpl::NewPoWValidBlock(const CBlockIndex *pindex, const std::sha LOCK(cs_main); - static int nHighestFastAnnounce = 0; - if (pindex->nHeight <= nHighestFastAnnounce) + if (pindex->nHeight <= m_highest_fast_announce) return; - nHighestFastAnnounce = pindex->nHeight; + m_highest_fast_announce = pindex->nHeight; bool fWitnessEnabled = DeploymentActiveAt(*pindex, m_chainparams.GetConsensus(), Consensus::DEPLOYMENT_SEGWIT); uint256 hashBlock(pblock->GetHash()); + const std::shared_future<CSerializedNetMsg> lazy_ser{ + std::async(std::launch::deferred, [&] { return msgMaker.Make(NetMsgType::CMPCTBLOCK, *pcmpctblock); })}; { - LOCK(cs_most_recent_block); - most_recent_block_hash = hashBlock; - most_recent_block = pblock; - most_recent_compact_block = pcmpctblock; - fWitnessesPresentInMostRecentCompactBlock = fWitnessEnabled; + LOCK(m_most_recent_block_mutex); + m_most_recent_block_hash = hashBlock; + m_most_recent_block = pblock; + m_most_recent_compact_block = pcmpctblock; + m_most_recent_compact_block_has_witnesses = fWitnessEnabled; } - m_connman.ForEachNode([this, &pcmpctblock, pindex, &msgMaker, fWitnessEnabled, &hashBlock](CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { + m_connman.ForEachNode([this, pindex, fWitnessEnabled, &lazy_ser, &hashBlock](CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { AssertLockHeld(::cs_main); - // TODO: Avoid the repeated-serialization here if (pnode->GetCommonVersion() < INVALID_CB_NO_BAN_VERSION || pnode->fDisconnect) return; ProcessBlockAvailability(pnode->GetId()); @@ -1620,7 +1668,9 @@ void PeerManagerImpl::NewPoWValidBlock(const CBlockIndex *pindex, const std::sha LogPrint(BCLog::NET, "%s sending header-and-ids %s to peer=%d\n", "PeerManager::NewPoWValidBlock", hashBlock.ToString(), pnode->GetId()); - m_connman.PushMessage(pnode, msgMaker.Make(NetMsgType::CMPCTBLOCK, *pcmpctblock)); + + const CSerializedNetMsg& ser_cmpctblock{lazy_ser.get()}; + m_connman.PushMessage(pnode, ser_cmpctblock.Copy()); state.pindexBestHeaderSent = pindex; } }); @@ -1742,22 +1792,17 @@ void PeerManagerImpl::SendPings() void PeerManagerImpl::RelayTransaction(const uint256& txid, const uint256& wtxid) { - WITH_LOCK(cs_main, _RelayTransaction(txid, wtxid);); -} - -void PeerManagerImpl::_RelayTransaction(const uint256& txid, const uint256& wtxid) -{ - m_connman.ForEachNode([&txid, &wtxid](CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { - AssertLockHeld(::cs_main); + LOCK(m_peer_mutex); + for(auto& it : m_peer_map) { + Peer& peer = *it.second; + if (!peer.m_tx_relay) continue; - CNodeState* state = State(pnode->GetId()); - if (state == nullptr) return; - if (state->m_wtxid_relay) { - pnode->PushTxInventory(wtxid); - } else { - pnode->PushTxInventory(txid); + const uint256& hash{peer.m_wtxid_relay ? wtxid : txid}; + LOCK(peer.m_tx_relay->m_tx_inventory_mutex); + if (!peer.m_tx_relay->m_tx_inventory_known_filter.contains(hash)) { + peer.m_tx_relay->m_tx_inventory_to_send.insert(hash); } - }); + }; } void PeerManagerImpl::RelayAddress(NodeId originator, @@ -1776,9 +1821,12 @@ void PeerManagerImpl::RelayAddress(NodeId originator, // Use deterministic randomness to send to the same nodes for 24 hours // at a time so the m_addr_knowns of the chosen nodes prevent repeats const uint64_t hash_addr{CServiceHash(0, 0)(addr)}; + const auto current_time{GetTime<std::chrono::seconds>()}; + // Adding address hash makes exact rotation time different per address, while preserving periodicity. + const uint64_t time_addr{(static_cast<uint64_t>(count_seconds(current_time)) + hash_addr) / count_seconds(ROTATE_ADDR_RELAY_DEST_INTERVAL)}; const CSipHasher hasher{m_connman.GetDeterministicRandomizer(RANDOMIZER_ID_ADDRESS_RELAY) .Write(hash_addr) - .Write((GetTime() + hash_addr) / (24 * 60 * 60))}; + .Write(time_addr)}; FastRandomContext insecure_rand; // Relay reachable addresses to 2 peers. Unreachable addresses are relayed randomly to 1 or 2 peers. @@ -1813,10 +1861,10 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& std::shared_ptr<const CBlockHeaderAndShortTxIDs> a_recent_compact_block; bool fWitnessesPresentInARecentCompactBlock; { - LOCK(cs_most_recent_block); - a_recent_block = most_recent_block; - a_recent_compact_block = most_recent_compact_block; - fWitnessesPresentInARecentCompactBlock = fWitnessesPresentInMostRecentCompactBlock; + LOCK(m_most_recent_block_mutex); + a_recent_block = m_most_recent_block; + a_recent_compact_block = m_most_recent_compact_block; + fWitnessesPresentInARecentCompactBlock = m_most_recent_compact_block_has_witnesses; } bool need_activate_chain = false; @@ -1854,7 +1902,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); // disconnect node in case we have reached the outbound limit for serving historical blocks if (m_connman.OutboundTargetReached(true) && - (((pindexBestHeader != nullptr) && (pindexBestHeader->GetBlockTime() - pindex->GetBlockTime() > HISTORICAL_BLOCK_AGE)) || inv.IsMsgFilteredBlk()) && + (((m_chainman.m_best_header != nullptr) && (m_chainman.m_best_header->GetBlockTime() - pindex->GetBlockTime() > HISTORICAL_BLOCK_AGE)) || inv.IsMsgFilteredBlk()) && !pfrom.HasPermission(NetPermissionFlags::Download) // nodes with the download permission may exceed target ) { LogPrint(BCLog::NET, "historical block serving limit reached, disconnect peer=%d\n", pfrom.GetId()); @@ -1903,11 +1951,11 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& } else if (inv.IsMsgFilteredBlk()) { bool sendMerkleBlock = false; CMerkleBlock merkleBlock; - if (pfrom.m_tx_relay != nullptr) { - LOCK(pfrom.m_tx_relay->cs_filter); - if (pfrom.m_tx_relay->pfilter) { + if (peer.m_tx_relay != nullptr) { + LOCK(peer.m_tx_relay->m_bloom_filter_mutex); + if (peer.m_tx_relay->m_bloom_filter) { sendMerkleBlock = true; - merkleBlock = CMerkleBlock(*pblock, *pfrom.m_tx_relay->pfilter); + merkleBlock = CMerkleBlock(*pblock, *peer.m_tx_relay->m_bloom_filter); } } if (sendMerkleBlock) { @@ -1996,7 +2044,7 @@ void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic const auto now{GetTime<std::chrono::seconds>()}; // Get last mempool request time - const auto mempool_req = pfrom.m_tx_relay != nullptr ? pfrom.m_tx_relay->m_last_mempool_req.load() : std::chrono::seconds::min(); + const auto mempool_req = peer.m_tx_relay != nullptr ? peer.m_tx_relay->m_last_mempool_req.load() : std::chrono::seconds::min(); // Process as many TX items from the front of the getdata queue as // possible, since they're common and it's efficient to batch process @@ -2009,7 +2057,7 @@ void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic const CInv &inv = *it++; - if (pfrom.m_tx_relay == nullptr) { + if (peer.m_tx_relay == nullptr) { // Ignore GETDATA requests for transactions from blocks-only peers. continue; } @@ -2037,7 +2085,7 @@ void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic } for (const uint256& parent_txid : parent_ids_to_add) { // Relaying a transaction with a recent but unconfirmed parent. - if (WITH_LOCK(pfrom.m_tx_relay->cs_tx_inventory, return !pfrom.m_tx_relay->filterInventoryKnown.contains(parent_txid))) { + if (WITH_LOCK(peer.m_tx_relay->m_tx_inventory_mutex, return !peer.m_tx_relay->m_tx_inventory_known_filter.contains(parent_txid))) { LOCK(cs_main); State(pfrom.GetId())->m_recently_announced_invs.insert(parent_txid); } @@ -2079,7 +2127,8 @@ void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic } } -static uint32_t GetFetchFlags(const CNode& pfrom) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { +uint32_t PeerManagerImpl::GetFetchFlags(const CNode& pfrom) const EXCLUSIVE_LOCKS_REQUIRED(cs_main) +{ uint32_t nFetchFlags = 0; if (State(pfrom.GetId())->fHaveWitness) { nFetchFlags |= MSG_WITNESS_FLAG; @@ -2131,12 +2180,12 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, const Peer& peer, // nUnconnectingHeaders gets reset back to 0. if (!m_chainman.m_blockman.LookupBlockIndex(headers[0].hashPrevBlock) && nCount < MAX_BLOCKS_TO_ANNOUNCE) { nodestate->nUnconnectingHeaders++; - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(pindexBestHeader), uint256())); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), uint256())); LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", - headers[0].GetHash().ToString(), - headers[0].hashPrevBlock.ToString(), - pindexBestHeader->nHeight, - pfrom.GetId(), nodestate->nUnconnectingHeaders); + headers[0].GetHash().ToString(), + headers[0].hashPrevBlock.ToString(), + m_chainman.m_best_header->nHeight, + pfrom.GetId(), nodestate->nUnconnectingHeaders); // Set hashLastUnknownBlock for this peer, so that if we // eventually get the headers - even from a different peer - // we can use this peer to download. @@ -2193,7 +2242,7 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, const Peer& peer, if (nCount == MAX_HEADERS_RESULTS) { // Headers message had its maximum size; the peer may have more headers. - // TODO: optimize: if pindexLast is an ancestor of m_chainman.ActiveChain().Tip or pindexBestHeader, continue + // TODO: optimize: if pindexLast is an ancestor of m_chainman.ActiveChain().Tip or m_chainman.m_best_header, continue // from there instead. LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom.GetId(), peer.m_starting_height); @@ -2317,7 +2366,7 @@ void PeerManagerImpl::ProcessOrphanTx(std::set<uint256>& orphan_work_set) if (result.m_result_type == MempoolAcceptResult::ResultType::VALID) { LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); - _RelayTransaction(orphanHash, porphanTx->GetWitnessHash()); + RelayTransaction(orphanHash, porphanTx->GetWitnessHash()); m_orphanage.AddChildrenToWorkSet(*porphanTx, orphan_work_set); m_orphanage.EraseTx(orphanHash); for (const CTransactionRef& removedTx : result.m_replaced_transactions.value()) { @@ -2633,7 +2682,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Inbound peers send us their version message when they connect. // We send our version message in response. if (pfrom.IsInboundConn()) { - PushNodeVersion(pfrom); + PushNodeVersion(pfrom, *peer); } // Change version @@ -2672,9 +2721,12 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // set nodes not capable of serving the complete blockchain history as "limited nodes" pfrom.m_limited_node = (!(nServices & NODE_NETWORK) && (nServices & NODE_NETWORK_LIMITED)); - if (pfrom.m_tx_relay != nullptr) { - LOCK(pfrom.m_tx_relay->cs_filter); - pfrom.m_tx_relay->fRelayTxes = fRelay; // set to true after we get the first filter* message + if (peer->m_tx_relay != nullptr) { + { + LOCK(peer->m_tx_relay->m_bloom_filter_mutex); + peer->m_tx_relay->m_relay_txs = fRelay; // set to true after we get the first filter* message + } + if (fRelay) pfrom.m_relays_txs = true; } if((nServices & NODE_WITNESS)) @@ -2685,8 +2737,10 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Potentially mark this peer as a preferred download peer. { - LOCK(cs_main); - UpdatePreferredDownload(pfrom, State(pfrom.GetId())); + LOCK(cs_main); + CNodeState* state = State(pfrom.GetId()); + state->fPreferredDownload = (!pfrom.IsInboundConn() || pfrom.HasPermission(NetPermissionFlags::NoBan)) && !pfrom.IsAddrFetchConn() && !pfrom.fClient; + m_num_preferred_download_peers += state->fPreferredDownload; } // Self advertisement & GETADDR logic @@ -2863,9 +2917,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } if (pfrom.GetCommonVersion() >= WTXID_RELAY_VERSION) { - LOCK(cs_main); - if (!State(pfrom.GetId())->m_wtxid_relay) { - State(pfrom.GetId())->m_wtxid_relay = true; + if (!peer->m_wtxid_relay) { + peer->m_wtxid_relay = true; m_wtxid_relay_peers++; } else { LogPrint(BCLog::NET, "ignoring duplicate wtxidrelay from peer=%d\n", pfrom.GetId()); @@ -3001,7 +3054,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Reject tx INVs when the -blocksonly setting is enabled, or this is a // block-relay-only peer - bool reject_tx_invs{m_ignore_incoming_txs || (pfrom.m_tx_relay == nullptr)}; + bool reject_tx_invs{m_ignore_incoming_txs || (peer->m_tx_relay == nullptr)}; // Allow peers with relay permission to send data other than blocks in blocks only mode if (pfrom.HasPermission(NetPermissionFlags::Relay)) { @@ -3019,7 +3072,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Ignore INVs that don't match wtxidrelay setting. // Note that orphan parent fetching always uses MSG_TX GETDATAs regardless of the wtxidrelay setting. // This is fine as no INV messages are involved in that process. - if (State(pfrom.GetId())->m_wtxid_relay) { + if (peer->m_wtxid_relay) { if (inv.IsMsgTx()) continue; } else { if (inv.IsMsgWtx()) continue; @@ -3048,7 +3101,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, const bool fAlreadyHave = AlreadyHaveTx(gtxid); LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom.GetId()); - pfrom.AddKnownTx(inv.hash); + AddKnownTx(*peer, inv.hash); if (!fAlreadyHave && !m_chainman.ActiveChainstate().IsInitialBlockDownload()) { AddTxAnnouncement(pfrom, gtxid, current_time); } @@ -3058,8 +3111,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } if (best_block != nullptr) { - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(pindexBestHeader), *best_block)); - LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, best_block->ToString(), pfrom.GetId()); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), *best_block)); + LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", m_chainman.m_best_header->nHeight, best_block->ToString(), pfrom.GetId()); } return; @@ -3110,8 +3163,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, { std::shared_ptr<const CBlock> a_recent_block; { - LOCK(cs_most_recent_block); - a_recent_block = most_recent_block; + LOCK(m_most_recent_block_mutex); + a_recent_block = m_most_recent_block; } BlockValidationState state; if (!m_chainman.ActiveChainstate().ActivateBestChain(state, a_recent_block)) { @@ -3162,10 +3215,10 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, std::shared_ptr<const CBlock> recent_block; { - LOCK(cs_most_recent_block); - if (most_recent_block_hash == req.blockhash) - recent_block = most_recent_block; - // Unlock cs_most_recent_block to avoid cs_main lock inversion + LOCK(m_most_recent_block_mutex); + if (m_most_recent_block_hash == req.blockhash) + recent_block = m_most_recent_block; + // Unlock m_most_recent_block_mutex to avoid cs_main lock inversion } if (recent_block) { SendBlockTransactions(pfrom, *recent_block, req); @@ -3278,8 +3331,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Stop processing the transaction early if // 1) We are in blocks only mode and peer has no relay permission // 2) This peer is a block-relay-only peer - if ((m_ignore_incoming_txs && !pfrom.HasPermission(NetPermissionFlags::Relay)) || (pfrom.m_tx_relay == nullptr)) - { + if ((m_ignore_incoming_txs && !pfrom.HasPermission(NetPermissionFlags::Relay)) || (peer->m_tx_relay == nullptr)) { LogPrint(BCLog::NET, "transaction sent in violation of protocol peer=%d\n", pfrom.GetId()); pfrom.fDisconnect = true; return; @@ -3297,21 +3349,19 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, const uint256& txid = ptx->GetHash(); const uint256& wtxid = ptx->GetWitnessHash(); - LOCK2(cs_main, g_cs_orphans); - - CNodeState* nodestate = State(pfrom.GetId()); - - const uint256& hash = nodestate->m_wtxid_relay ? wtxid : txid; - pfrom.AddKnownTx(hash); - if (nodestate->m_wtxid_relay && txid != wtxid) { - // Insert txid into filterInventoryKnown, even for + const uint256& hash = peer->m_wtxid_relay ? wtxid : txid; + AddKnownTx(*peer, hash); + if (peer->m_wtxid_relay && txid != wtxid) { + // Insert txid into m_tx_inventory_known_filter, even for // wtxidrelay peers. This prevents re-adding of // unconfirmed parents to the recently_announced // filter, when a child tx is requested. See // ProcessGetData(). - pfrom.AddKnownTx(txid); + AddKnownTx(*peer, txid); } + LOCK2(cs_main, g_cs_orphans); + m_txrequest.ReceivedResponse(pfrom.GetId(), txid); if (tx.HasWitness()) m_txrequest.ReceivedResponse(pfrom.GetId(), wtxid); @@ -3336,7 +3386,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, LogPrintf("Not relaying non-mempool transaction %s from forcerelay peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); } else { LogPrintf("Force relaying tx %s from peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); - _RelayTransaction(tx.GetHash(), tx.GetWitnessHash()); + RelayTransaction(tx.GetHash(), tx.GetWitnessHash()); } } return; @@ -3350,7 +3400,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // requests for it. m_txrequest.ForgetTxHash(tx.GetHash()); m_txrequest.ForgetTxHash(tx.GetWitnessHash()); - _RelayTransaction(tx.GetHash(), tx.GetWitnessHash()); + RelayTransaction(tx.GetHash(), tx.GetWitnessHash()); m_orphanage.AddChildrenToWorkSet(tx, peer->m_orphan_work_set); pfrom.m_last_tx_time = GetTime<std::chrono::seconds>(); @@ -3397,7 +3447,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Eventually we should replace this with an improved // protocol for getting all unconfirmed parents. const auto gtxid{GenTxid::Txid(parent_txid)}; - pfrom.AddKnownTx(parent_txid); + AddKnownTx(*peer, parent_txid); if (!AlreadyHaveTx(gtxid)) AddTxAnnouncement(pfrom, gtxid, current_time); } @@ -3508,7 +3558,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (!m_chainman.m_blockman.LookupBlockIndex(cmpctblock.header.hashPrevBlock)) { // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers if (!m_chainman.ActiveChainstate().IsInitialBlockDownload()) - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(pindexBestHeader), uint256())); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), uint256())); return; } @@ -3521,7 +3571,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, BlockValidationState state; if (!m_chainman.ProcessNewBlockHeaders({cmpctblock.header}, state, m_chainparams, &pindex)) { if (state.IsInvalid()) { - MaybePunishNodeForBlock(pfrom.GetId(), state, /*via_compact_block*/ true, "invalid header via cmpctblock"); + MaybePunishNodeForBlock(pfrom.GetId(), state, /*via_compact_block=*/true, "invalid header via cmpctblock"); return; } } @@ -3861,7 +3911,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, peer->m_addrs_to_send.clear(); std::vector<CAddress> vAddr; if (pfrom.HasPermission(NetPermissionFlags::Addr)) { - vAddr = m_connman.GetAddresses(MAX_ADDR_TO_SEND, MAX_PCT_ADDR_TO_SEND, /* network */ std::nullopt); + vAddr = m_connman.GetAddresses(MAX_ADDR_TO_SEND, MAX_PCT_ADDR_TO_SEND, /*network=*/std::nullopt); } else { vAddr = m_connman.GetAddresses(pfrom, MAX_ADDR_TO_SEND, MAX_PCT_ADDR_TO_SEND); } @@ -3893,9 +3943,9 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } - if (pfrom.m_tx_relay != nullptr) { - LOCK(pfrom.m_tx_relay->cs_tx_inventory); - pfrom.m_tx_relay->fSendMempool = true; + if (peer->m_tx_relay != nullptr) { + LOCK(peer->m_tx_relay->m_tx_inventory_mutex); + peer->m_tx_relay->m_send_mempool = true; } return; } @@ -3989,11 +4039,15 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // There is no excuse for sending a too-large filter Misbehaving(pfrom.GetId(), 100, "too-large bloom filter"); } - else if (pfrom.m_tx_relay != nullptr) + else if (peer->m_tx_relay != nullptr) { - LOCK(pfrom.m_tx_relay->cs_filter); - pfrom.m_tx_relay->pfilter.reset(new CBloomFilter(filter)); - pfrom.m_tx_relay->fRelayTxes = true; + { + LOCK(peer->m_tx_relay->m_bloom_filter_mutex); + peer->m_tx_relay->m_bloom_filter.reset(new CBloomFilter(filter)); + peer->m_tx_relay->m_relay_txs = true; + } + pfrom.m_bloom_filter_loaded = true; + pfrom.m_relays_txs = true; } return; } @@ -4012,10 +4066,10 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, bool bad = false; if (vData.size() > MAX_SCRIPT_ELEMENT_SIZE) { bad = true; - } else if (pfrom.m_tx_relay != nullptr) { - LOCK(pfrom.m_tx_relay->cs_filter); - if (pfrom.m_tx_relay->pfilter) { - pfrom.m_tx_relay->pfilter->insert(vData); + } else if (peer->m_tx_relay != nullptr) { + LOCK(peer->m_tx_relay->m_bloom_filter_mutex); + if (peer->m_tx_relay->m_bloom_filter) { + peer->m_tx_relay->m_bloom_filter->insert(vData); } else { bad = true; } @@ -4032,12 +4086,17 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, pfrom.fDisconnect = true; return; } - if (pfrom.m_tx_relay == nullptr) { + if (peer->m_tx_relay == nullptr) { return; } - LOCK(pfrom.m_tx_relay->cs_filter); - pfrom.m_tx_relay->pfilter = nullptr; - pfrom.m_tx_relay->fRelayTxes = true; + + { + LOCK(peer->m_tx_relay->m_bloom_filter_mutex); + peer->m_tx_relay->m_bloom_filter = nullptr; + peer->m_tx_relay->m_relay_txs = true; + } + pfrom.m_bloom_filter_loaded = false; + pfrom.m_relays_txs = true; return; } @@ -4045,8 +4104,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, CAmount newFeeFilter = 0; vRecv >> newFeeFilter; if (MoneyRange(newFeeFilter)) { - if (pfrom.m_tx_relay != nullptr) { - pfrom.m_tx_relay->minFeeFilter = newFeeFilter; + if (peer->m_tx_relay != nullptr) { + peer->m_tx_relay->m_fee_filter_received = newFeeFilter; } LogPrint(BCLog::NET, "received: feefilter of %s from peer=%d\n", CFeeRate(newFeeFilter).ToString(), pfrom.GetId()); } @@ -4422,7 +4481,7 @@ void PeerManagerImpl::MaybeSendPing(CNode& node_to, Peer& peer, std::chrono::mic if (pingSend) { uint64_t nonce = 0; while (nonce == 0) { - GetRandBytes((unsigned char*)&nonce, sizeof(nonce)); + GetRandBytes({(unsigned char*)&nonce, sizeof(nonce)}); } peer.m_ping_queued = false; peer.m_ping_start = now; @@ -4504,12 +4563,10 @@ void PeerManagerImpl::MaybeSendAddr(CNode& node, Peer& peer, std::chrono::micros } } -void PeerManagerImpl::MaybeSendFeefilter(CNode& pto, std::chrono::microseconds current_time) +void PeerManagerImpl::MaybeSendFeefilter(CNode& pto, Peer& peer, std::chrono::microseconds current_time) { - AssertLockHeld(cs_main); - if (m_ignore_incoming_txs) return; - if (!pto.m_tx_relay) return; + if (!peer.m_tx_relay) return; if (pto.GetCommonVersion() < FEEFILTER_VERSION) return; // peers with the forcerelay permission should not filter txs to us if (pto.HasPermission(NetPermissionFlags::ForceRelay)) return; @@ -4523,27 +4580,27 @@ void PeerManagerImpl::MaybeSendFeefilter(CNode& pto, std::chrono::microseconds c currentFilter = MAX_MONEY; } else { static const CAmount MAX_FILTER{g_filter_rounder.round(MAX_MONEY)}; - if (pto.m_tx_relay->lastSentFeeFilter == MAX_FILTER) { + if (peer.m_tx_relay->m_fee_filter_sent == MAX_FILTER) { // Send the current filter if we sent MAX_FILTER previously // and made it out of IBD. - pto.m_tx_relay->m_next_send_feefilter = 0us; + peer.m_tx_relay->m_next_send_feefilter = 0us; } } - if (current_time > pto.m_tx_relay->m_next_send_feefilter) { + if (current_time > peer.m_tx_relay->m_next_send_feefilter) { CAmount filterToSend = g_filter_rounder.round(currentFilter); // We always have a fee filter of at least minRelayTxFee filterToSend = std::max(filterToSend, ::minRelayTxFee.GetFeePerK()); - if (filterToSend != pto.m_tx_relay->lastSentFeeFilter) { + if (filterToSend != peer.m_tx_relay->m_fee_filter_sent) { m_connman.PushMessage(&pto, CNetMsgMaker(pto.GetCommonVersion()).Make(NetMsgType::FEEFILTER, filterToSend)); - pto.m_tx_relay->lastSentFeeFilter = filterToSend; + peer.m_tx_relay->m_fee_filter_sent = filterToSend; } - pto.m_tx_relay->m_next_send_feefilter = GetExponentialRand(current_time, AVG_FEEFILTER_BROADCAST_INTERVAL); + peer.m_tx_relay->m_next_send_feefilter = GetExponentialRand(current_time, AVG_FEEFILTER_BROADCAST_INTERVAL); } // If the fee filter has changed substantially and it's still more than MAX_FEEFILTER_CHANGE_DELAY // until scheduled broadcast, then move the broadcast to within MAX_FEEFILTER_CHANGE_DELAY. - else if (current_time + MAX_FEEFILTER_CHANGE_DELAY < pto.m_tx_relay->m_next_send_feefilter && - (currentFilter < 3 * pto.m_tx_relay->lastSentFeeFilter / 4 || currentFilter > 4 * pto.m_tx_relay->lastSentFeeFilter / 3)) { - pto.m_tx_relay->m_next_send_feefilter = current_time + GetRandomDuration<std::chrono::microseconds>(MAX_FEEFILTER_CHANGE_DELAY); + else if (current_time + MAX_FEEFILTER_CHANGE_DELAY < peer.m_tx_relay->m_next_send_feefilter && + (currentFilter < 3 * peer.m_tx_relay->m_fee_filter_sent / 4 || currentFilter > 4 * peer.m_tx_relay->m_fee_filter_sent / 3)) { + peer.m_tx_relay->m_next_send_feefilter = current_time + GetRandomDuration<std::chrono::microseconds>(MAX_FEEFILTER_CHANGE_DELAY); } } @@ -4622,28 +4679,29 @@ bool PeerManagerImpl::SendMessages(CNode* pto) CNodeState &state = *State(pto->GetId()); // Start block sync - if (pindexBestHeader == nullptr) - pindexBestHeader = m_chainman.ActiveChain().Tip(); - bool fFetch = state.fPreferredDownload || (nPreferredDownload == 0 && !pto->fClient && !pto->IsAddrFetchConn()); // Download if this is a nice peer, or we have no nice peers and this one might do. + if (m_chainman.m_best_header == nullptr) { + m_chainman.m_best_header = m_chainman.ActiveChain().Tip(); + } + bool fFetch = state.fPreferredDownload || (m_num_preferred_download_peers == 0 && !pto->fClient && !pto->IsAddrFetchConn()); // Download if this is a nice peer, or we have no nice peers and this one might do. if (!state.fSyncStarted && !pto->fClient && !fImporting && !fReindex) { // Only actively request headers from a single peer, unless we're close to today. - if ((nSyncStarted == 0 && fFetch) || pindexBestHeader->GetBlockTime() > GetAdjustedTime() - 24 * 60 * 60) { + if ((nSyncStarted == 0 && fFetch) || m_chainman.m_best_header->GetBlockTime() > GetAdjustedTime() - 24 * 60 * 60) { state.fSyncStarted = true; state.m_headers_sync_timeout = current_time + HEADERS_DOWNLOAD_TIMEOUT_BASE + ( // Convert HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER to microseconds before scaling // to maintain precision std::chrono::microseconds{HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER} * - (GetAdjustedTime() - pindexBestHeader->GetBlockTime()) / consensusParams.nPowTargetSpacing + (GetAdjustedTime() - m_chainman.m_best_header->GetBlockTime()) / consensusParams.nPowTargetSpacing ); nSyncStarted++; - const CBlockIndex *pindexStart = pindexBestHeader; + const CBlockIndex* pindexStart = m_chainman.m_best_header; /* If possible, start at the block preceding the currently best known header. This ensures that we always get a non-empty list of headers back as long as the peer is up-to-date. With a non-empty response, we can initialise the peer's known best block. This wouldn't be possible - if we requested starting at pindexBestHeader and + if we requested starting at m_chainman.m_best_header and got back an empty response. */ if (pindexStart->pprev) pindexStart = pindexStart->pprev; @@ -4729,12 +4787,12 @@ bool PeerManagerImpl::SendMessages(CNode* pto) bool fGotBlockFromCache = false; { - LOCK(cs_most_recent_block); - if (most_recent_block_hash == pBestIndex->GetBlockHash()) { - if (state.fWantsCmpctWitness || !fWitnessesPresentInMostRecentCompactBlock) - m_connman.PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *most_recent_compact_block)); + LOCK(m_most_recent_block_mutex); + if (m_most_recent_block_hash == pBestIndex->GetBlockHash()) { + if (state.fWantsCmpctWitness || !m_most_recent_compact_block_has_witnesses) + m_connman.PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *m_most_recent_compact_block)); else { - CBlockHeaderAndShortTxIDs cmpctblock(*most_recent_block, state.fWantsCmpctWitness); + CBlockHeaderAndShortTxIDs cmpctblock(*m_most_recent_block, state.fWantsCmpctWitness); m_connman.PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); } fGotBlockFromCache = true; @@ -4810,45 +4868,45 @@ bool PeerManagerImpl::SendMessages(CNode* pto) peer->m_blocks_for_inv_relay.clear(); } - if (pto->m_tx_relay != nullptr) { - LOCK(pto->m_tx_relay->cs_tx_inventory); + if (peer->m_tx_relay != nullptr) { + LOCK(peer->m_tx_relay->m_tx_inventory_mutex); // Check whether periodic sends should happen bool fSendTrickle = pto->HasPermission(NetPermissionFlags::NoBan); - if (pto->m_tx_relay->nNextInvSend < current_time) { + if (peer->m_tx_relay->m_next_inv_send_time < current_time) { fSendTrickle = true; if (pto->IsInboundConn()) { - pto->m_tx_relay->nNextInvSend = NextInvToInbounds(current_time, INBOUND_INVENTORY_BROADCAST_INTERVAL); + peer->m_tx_relay->m_next_inv_send_time = NextInvToInbounds(current_time, INBOUND_INVENTORY_BROADCAST_INTERVAL); } else { - pto->m_tx_relay->nNextInvSend = GetExponentialRand(current_time, OUTBOUND_INVENTORY_BROADCAST_INTERVAL); + peer->m_tx_relay->m_next_inv_send_time = GetExponentialRand(current_time, OUTBOUND_INVENTORY_BROADCAST_INTERVAL); } } // Time to send but the peer has requested we not relay transactions. if (fSendTrickle) { - LOCK(pto->m_tx_relay->cs_filter); - if (!pto->m_tx_relay->fRelayTxes) pto->m_tx_relay->setInventoryTxToSend.clear(); + LOCK(peer->m_tx_relay->m_bloom_filter_mutex); + if (!peer->m_tx_relay->m_relay_txs) peer->m_tx_relay->m_tx_inventory_to_send.clear(); } // Respond to BIP35 mempool requests - if (fSendTrickle && pto->m_tx_relay->fSendMempool) { + if (fSendTrickle && peer->m_tx_relay->m_send_mempool) { auto vtxinfo = m_mempool.infoAll(); - pto->m_tx_relay->fSendMempool = false; - const CFeeRate filterrate{pto->m_tx_relay->minFeeFilter.load()}; + peer->m_tx_relay->m_send_mempool = false; + const CFeeRate filterrate{peer->m_tx_relay->m_fee_filter_received.load()}; - LOCK(pto->m_tx_relay->cs_filter); + LOCK(peer->m_tx_relay->m_bloom_filter_mutex); for (const auto& txinfo : vtxinfo) { - const uint256& hash = state.m_wtxid_relay ? txinfo.tx->GetWitnessHash() : txinfo.tx->GetHash(); - CInv inv(state.m_wtxid_relay ? MSG_WTX : MSG_TX, hash); - pto->m_tx_relay->setInventoryTxToSend.erase(hash); + const uint256& hash = peer->m_wtxid_relay ? txinfo.tx->GetWitnessHash() : txinfo.tx->GetHash(); + CInv inv(peer->m_wtxid_relay ? MSG_WTX : MSG_TX, hash); + peer->m_tx_relay->m_tx_inventory_to_send.erase(hash); // Don't send transactions that peers will not put into their mempool if (txinfo.fee < filterrate.GetFee(txinfo.vsize)) { continue; } - if (pto->m_tx_relay->pfilter) { - if (!pto->m_tx_relay->pfilter->IsRelevantAndUpdate(*txinfo.tx)) continue; + if (peer->m_tx_relay->m_bloom_filter) { + if (!peer->m_tx_relay->m_bloom_filter->IsRelevantAndUpdate(*txinfo.tx)) continue; } - pto->m_tx_relay->filterInventoryKnown.insert(hash); + peer->m_tx_relay->m_tx_inventory_known_filter.insert(hash); // Responses to MEMPOOL requests bypass the m_recently_announced_invs filter. vInv.push_back(inv); if (vInv.size() == MAX_INV_SZ) { @@ -4856,37 +4914,37 @@ bool PeerManagerImpl::SendMessages(CNode* pto) vInv.clear(); } } - pto->m_tx_relay->m_last_mempool_req = std::chrono::duration_cast<std::chrono::seconds>(current_time); + peer->m_tx_relay->m_last_mempool_req = std::chrono::duration_cast<std::chrono::seconds>(current_time); } // Determine transactions to relay if (fSendTrickle) { // Produce a vector with all candidates for sending std::vector<std::set<uint256>::iterator> vInvTx; - vInvTx.reserve(pto->m_tx_relay->setInventoryTxToSend.size()); - for (std::set<uint256>::iterator it = pto->m_tx_relay->setInventoryTxToSend.begin(); it != pto->m_tx_relay->setInventoryTxToSend.end(); it++) { + vInvTx.reserve(peer->m_tx_relay->m_tx_inventory_to_send.size()); + for (std::set<uint256>::iterator it = peer->m_tx_relay->m_tx_inventory_to_send.begin(); it != peer->m_tx_relay->m_tx_inventory_to_send.end(); it++) { vInvTx.push_back(it); } - const CFeeRate filterrate{pto->m_tx_relay->minFeeFilter.load()}; + const CFeeRate filterrate{peer->m_tx_relay->m_fee_filter_received.load()}; // Topologically and fee-rate sort the inventory we send for privacy and priority reasons. // A heap is used so that not all items need sorting if only a few are being sent. - CompareInvMempoolOrder compareInvMempoolOrder(&m_mempool, state.m_wtxid_relay); + CompareInvMempoolOrder compareInvMempoolOrder(&m_mempool, peer->m_wtxid_relay); std::make_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder); // No reason to drain out at many times the network's capacity, // especially since we have many peers and some will draw much shorter delays. unsigned int nRelayedTransactions = 0; - LOCK(pto->m_tx_relay->cs_filter); + LOCK(peer->m_tx_relay->m_bloom_filter_mutex); while (!vInvTx.empty() && nRelayedTransactions < INVENTORY_BROADCAST_MAX) { // Fetch the top element from the heap std::pop_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder); std::set<uint256>::iterator it = vInvTx.back(); vInvTx.pop_back(); uint256 hash = *it; - CInv inv(state.m_wtxid_relay ? MSG_WTX : MSG_TX, hash); + CInv inv(peer->m_wtxid_relay ? MSG_WTX : MSG_TX, hash); // Remove it from the to-be-sent set - pto->m_tx_relay->setInventoryTxToSend.erase(it); + peer->m_tx_relay->m_tx_inventory_to_send.erase(it); // Check if not in the filter already - if (pto->m_tx_relay->filterInventoryKnown.contains(hash)) { + if (peer->m_tx_relay->m_tx_inventory_known_filter.contains(hash)) { continue; } // Not in the mempool anymore? don't bother sending it. @@ -4900,7 +4958,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) if (txinfo.fee < filterrate.GetFee(txinfo.vsize)) { continue; } - if (pto->m_tx_relay->pfilter && !pto->m_tx_relay->pfilter->IsRelevantAndUpdate(*txinfo.tx)) continue; + if (peer->m_tx_relay->m_bloom_filter && !peer->m_tx_relay->m_bloom_filter->IsRelevantAndUpdate(*txinfo.tx)) continue; // Send State(pto->GetId())->m_recently_announced_invs.insert(hash); vInv.push_back(inv); @@ -4927,14 +4985,14 @@ bool PeerManagerImpl::SendMessages(CNode* pto) m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } - pto->m_tx_relay->filterInventoryKnown.insert(hash); + peer->m_tx_relay->m_tx_inventory_known_filter.insert(hash); if (hash != txid) { - // Insert txid into filterInventoryKnown, even for + // Insert txid into m_tx_inventory_known_filter, even for // wtxidrelay peers. This prevents re-adding of // unconfirmed parents to the recently_announced // filter, when a child tx is requested. See // ProcessGetData(). - pto->m_tx_relay->filterInventoryKnown.insert(txid); + peer->m_tx_relay->m_tx_inventory_known_filter.insert(txid); } } } @@ -4968,8 +5026,8 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // Check for headers sync timeouts if (state.fSyncStarted && state.m_headers_sync_timeout < std::chrono::microseconds::max()) { // Detect whether this is a stalling initial-headers-sync peer - if (pindexBestHeader->GetBlockTime() <= GetAdjustedTime() - 24 * 60 * 60) { - if (current_time > state.m_headers_sync_timeout && nSyncStarted == 1 && (nPreferredDownload - state.fPreferredDownload >= 1)) { + if (m_chainman.m_best_header->GetBlockTime() <= GetAdjustedTime() - 24 * 60 * 60) { + if (current_time > state.m_headers_sync_timeout && nSyncStarted == 1 && (m_num_preferred_download_peers - state.fPreferredDownload >= 1)) { // Disconnect a peer (without NetPermissionFlags::NoBan permission) if it is our only sync peer, // and we have others we could be using instead. // Note: If all our peers are inbound, then we won't @@ -5054,8 +5112,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) if (!vGetData.empty()) m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); - - MaybeSendFeefilter(*pto, current_time); } // release cs_main + MaybeSendFeefilter(*pto, *peer, current_time); return true; } diff --git a/src/net_processing.h b/src/net_processing.h index e30f9f516c..c982b919a6 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -29,6 +29,8 @@ struct CNodeStateStats { int m_starting_height = -1; std::chrono::microseconds m_ping_wait; std::vector<int> vHeightInFlight; + bool m_relay_txs; + CAmount m_fee_filter_received; uint64_t m_addr_processed = 0; uint64_t m_addr_rate_limited = 0; bool m_addr_relay_enabled{false}; @@ -85,6 +87,9 @@ public: /** Process a single message from a peer. Public for fuzz testing */ virtual void ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, const std::chrono::microseconds time_received, const std::atomic<bool>& interruptMsgProc) = 0; + + /** This function is used for testing the stale tip eviction logic, see denialofservice_tests.cpp */ + virtual void UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_seconds) = 0; }; #endif // BITCOIN_NET_PROCESSING_H diff --git a/src/netaddress.cpp b/src/netaddress.cpp index f7640329f8..9717f7abce 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -10,7 +10,6 @@ #include <hash.h> #include <prevector.h> #include <tinyformat.h> -#include <util/asmap.h> #include <util/strencodings.h> #include <util/string.h> @@ -21,9 +20,6 @@ #include <iterator> #include <tuple> -constexpr size_t CNetAddr::V1_SERIALIZATION_SIZE; -constexpr size_t CNetAddr::MAX_ADDRV2_SIZE; - CNetAddr::BIP155Network CNetAddr::GetBIP155Network() const { switch (m_net) { @@ -211,7 +207,7 @@ static void Checksum(Span<const uint8_t> addr_pubkey, uint8_t (&checksum)[CHECKS bool CNetAddr::SetSpecial(const std::string& addr) { - if (!ValidAsCString(addr)) { + if (!ContainsNoNUL(addr)) { return false; } @@ -235,17 +231,16 @@ bool CNetAddr::SetTor(const std::string& addr) return false; } - bool invalid; - const auto& input = DecodeBase32(addr.substr(0, addr.size() - suffix_len).c_str(), &invalid); + auto input = DecodeBase32(std::string_view{addr}.substr(0, addr.size() - suffix_len)); - if (invalid) { + if (!input) { return false; } - if (input.size() == torv3::TOTAL_LEN) { - Span<const uint8_t> input_pubkey{input.data(), ADDR_TORV3_SIZE}; - Span<const uint8_t> input_checksum{input.data() + ADDR_TORV3_SIZE, torv3::CHECKSUM_LEN}; - Span<const uint8_t> input_version{input.data() + ADDR_TORV3_SIZE + torv3::CHECKSUM_LEN, sizeof(torv3::VERSION)}; + if (input->size() == torv3::TOTAL_LEN) { + Span<const uint8_t> input_pubkey{input->data(), ADDR_TORV3_SIZE}; + Span<const uint8_t> input_checksum{input->data() + ADDR_TORV3_SIZE, torv3::CHECKSUM_LEN}; + Span<const uint8_t> input_version{input->data() + ADDR_TORV3_SIZE + torv3::CHECKSUM_LEN, sizeof(torv3::VERSION)}; if (input_version != torv3::VERSION) { return false; @@ -281,15 +276,14 @@ bool CNetAddr::SetI2P(const std::string& addr) // can decode it. const std::string b32_padded = addr.substr(0, b32_len) + "===="; - bool invalid; - const auto& address_bytes = DecodeBase32(b32_padded.c_str(), &invalid); + auto address_bytes = DecodeBase32(b32_padded); - if (invalid || address_bytes.size() != ADDR_I2P_SIZE) { + if (!address_bytes || address_bytes->size() != ADDR_I2P_SIZE) { return false; } m_net = NET_I2P; - m_addr.assign(address_bytes.begin(), address_bytes.end()); + m_addr.assign(address_bytes->begin(), address_bytes->end()); return true; } @@ -722,107 +716,6 @@ Network CNetAddr::GetNetClass() const return m_net; } -uint32_t CNetAddr::GetMappedAS(const std::vector<bool> &asmap) const { - uint32_t net_class = GetNetClass(); - if (asmap.size() == 0 || (net_class != NET_IPV4 && net_class != NET_IPV6)) { - return 0; // Indicates not found, safe because AS0 is reserved per RFC7607. - } - std::vector<bool> ip_bits(128); - if (HasLinkedIPv4()) { - // For lookup, treat as if it was just an IPv4 address (IPV4_IN_IPV6_PREFIX + IPv4 bits) - for (int8_t byte_i = 0; byte_i < 12; ++byte_i) { - for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { - ip_bits[byte_i * 8 + bit_i] = (IPV4_IN_IPV6_PREFIX[byte_i] >> (7 - bit_i)) & 1; - } - } - uint32_t ipv4 = GetLinkedIPv4(); - for (int i = 0; i < 32; ++i) { - ip_bits[96 + i] = (ipv4 >> (31 - i)) & 1; - } - } else { - // Use all 128 bits of the IPv6 address otherwise - assert(IsIPv6()); - for (int8_t byte_i = 0; byte_i < 16; ++byte_i) { - uint8_t cur_byte = m_addr[byte_i]; - for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { - ip_bits[byte_i * 8 + bit_i] = (cur_byte >> (7 - bit_i)) & 1; - } - } - } - uint32_t mapped_as = Interpret(asmap, ip_bits); - return mapped_as; -} - -/** - * Get the canonical identifier of our network group - * - * The groups are assigned in a way where it should be costly for an attacker to - * obtain addresses with many different group identifiers, even if it is cheap - * to obtain addresses with the same identifier. - * - * @note No two connections will be attempted to addresses with the same network - * group. - */ -std::vector<unsigned char> CNetAddr::GetGroup(const std::vector<bool> &asmap) const -{ - std::vector<unsigned char> vchRet; - uint32_t net_class = GetNetClass(); - // If non-empty asmap is supplied and the address is IPv4/IPv6, - // return ASN to be used for bucketing. - uint32_t asn = GetMappedAS(asmap); - if (asn != 0) { // Either asmap was empty, or address has non-asmappable net class (e.g. TOR). - vchRet.push_back(NET_IPV6); // IPv4 and IPv6 with same ASN should be in the same bucket - for (int i = 0; i < 4; i++) { - vchRet.push_back((asn >> (8 * i)) & 0xFF); - } - return vchRet; - } - - vchRet.push_back(net_class); - int nBits{0}; - - if (IsLocal()) { - // all local addresses belong to the same group - } else if (IsInternal()) { - // all internal-usage addresses get their own group - nBits = ADDR_INTERNAL_SIZE * 8; - } else if (!IsRoutable()) { - // all other unroutable addresses belong to the same group - } else if (HasLinkedIPv4()) { - // IPv4 addresses (and mapped IPv4 addresses) use /16 groups - uint32_t ipv4 = GetLinkedIPv4(); - vchRet.push_back((ipv4 >> 24) & 0xFF); - vchRet.push_back((ipv4 >> 16) & 0xFF); - return vchRet; - } else if (IsTor() || IsI2P()) { - nBits = 4; - } else if (IsCJDNS()) { - // Treat in the same way as Tor and I2P because the address in all of - // them is "random" bytes (derived from a public key). However in CJDNS - // the first byte is a constant 0xfc, so the random bytes come after it. - // Thus skip the constant 8 bits at the start. - nBits = 12; - } else if (IsHeNet()) { - // for he.net, use /36 groups - nBits = 36; - } else { - // for the rest of the IPv6 network, use /32 groups - nBits = 32; - } - - // Push our address onto vchRet. - const size_t num_bytes = nBits / 8; - vchRet.insert(vchRet.end(), m_addr.begin(), m_addr.begin() + num_bytes); - nBits %= 8; - // ...for the last byte, push nBits and for the rest of the byte push 1's - if (nBits > 0) { - assert(num_bytes < m_addr.size()); - vchRet.push_back(m_addr[num_bytes] | ((1 << (8 - nBits)) - 1)); - } - - return vchRet; -} - std::vector<unsigned char> CNetAddr::GetAddrBytes() const { if (IsAddrV1Compatible()) { diff --git a/src/netaddress.h b/src/netaddress.h index 6d21dcd5cd..b9a8dc589a 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -202,12 +202,6 @@ public: //! Whether this address has a linked IPv4 address (see GetLinkedIPv4()). bool HasLinkedIPv4() const; - // The AS on the BGP path to the node we use to diversify - // peers in AddrMan bucketing based on the AS infrastructure. - // The ip->AS mapping depends on how asmap is constructed. - uint32_t GetMappedAS(const std::vector<bool>& asmap) const; - - std::vector<unsigned char> GetGroup(const std::vector<bool>& asmap) const; std::vector<unsigned char> GetAddrBytes() const; int GetReachabilityFrom(const CNetAddr* paddrPartner = nullptr) const; diff --git a/src/netbase.cpp b/src/netbase.cpp index 9a0b800565..8ff3b7a68c 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -136,7 +136,7 @@ static bool LookupIntern(const std::string& name, std::vector<CNetAddr>& vIP, un { vIP.clear(); - if (!ValidAsCString(name)) { + if (!ContainsNoNUL(name)) { return false; } @@ -169,7 +169,7 @@ static bool LookupIntern(const std::string& name, std::vector<CNetAddr>& vIP, un bool LookupHost(const std::string& name, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup, DNSLookupFn dns_lookup_function) { - if (!ValidAsCString(name)) { + if (!ContainsNoNUL(name)) { return false; } std::string strHost = name; @@ -184,7 +184,7 @@ bool LookupHost(const std::string& name, std::vector<CNetAddr>& vIP, unsigned in bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup, DNSLookupFn dns_lookup_function) { - if (!ValidAsCString(name)) { + if (!ContainsNoNUL(name)) { return false; } std::vector<CNetAddr> vIP; @@ -197,7 +197,7 @@ bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup, DNSL bool Lookup(const std::string& name, std::vector<CService>& vAddr, uint16_t portDefault, bool fAllowLookup, unsigned int nMaxSolutions, DNSLookupFn dns_lookup_function) { - if (name.empty() || !ValidAsCString(name)) { + if (name.empty() || !ContainsNoNUL(name)) { return false; } uint16_t port{portDefault}; @@ -216,7 +216,7 @@ bool Lookup(const std::string& name, std::vector<CService>& vAddr, uint16_t port bool Lookup(const std::string& name, CService& addr, uint16_t portDefault, bool fAllowLookup, DNSLookupFn dns_lookup_function) { - if (!ValidAsCString(name)) { + if (!ContainsNoNUL(name)) { return false; } std::vector<CService> vService; @@ -229,7 +229,7 @@ bool Lookup(const std::string& name, CService& addr, uint16_t portDefault, bool CService LookupNumeric(const std::string& name, uint16_t portDefault, DNSLookupFn dns_lookup_function) { - if (!ValidAsCString(name)) { + if (!ContainsNoNUL(name)) { return {}; } CService addr; @@ -305,7 +305,7 @@ enum class IntrRecvError { * * @see This function can be interrupted by calling InterruptSocks5(bool). * Sockets can be made non-blocking with SetSocketNonBlocking(const - * SOCKET&, bool). + * SOCKET&). */ static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, int timeout, const Sock& sock) { @@ -499,10 +499,11 @@ std::unique_ptr<Sock> CreateSockTCP(const CService& address_family) return nullptr; } + auto sock = std::make_unique<Sock>(hSocket); + // Ensure that waiting for I/O on this socket won't result in undefined // behavior. - if (!IsSelectableSocket(hSocket)) { - CloseSocket(hSocket); + if (!IsSelectableSocket(sock->Get())) { LogPrintf("Cannot create connection: non-selectable socket created (fd >= FD_SETSIZE ?)\n"); return nullptr; } @@ -511,19 +512,24 @@ std::unique_ptr<Sock> CreateSockTCP(const CService& address_family) int set = 1; // Set the no-sigpipe option on the socket for BSD systems, other UNIXes // should use the MSG_NOSIGNAL flag for every send. - setsockopt(hSocket, SOL_SOCKET, SO_NOSIGPIPE, (void*)&set, sizeof(int)); + if (sock->SetSockOpt(SOL_SOCKET, SO_NOSIGPIPE, (void*)&set, sizeof(int)) == SOCKET_ERROR) { + LogPrintf("Error setting SO_NOSIGPIPE on socket: %s, continuing anyway\n", + NetworkErrorString(WSAGetLastError())); + } #endif // Set the no-delay option (disable Nagle's algorithm) on the TCP socket. - SetSocketNoDelay(hSocket); + const int on{1}; + if (sock->SetSockOpt(IPPROTO_TCP, TCP_NODELAY, &on, sizeof(on)) == SOCKET_ERROR) { + LogPrint(BCLog::NET, "Unable to set TCP_NODELAY on a newly created socket, continuing anyway\n"); + } // Set the non-blocking option on the socket. - if (!SetSocketNonBlocking(hSocket, true)) { - CloseSocket(hSocket); + if (!SetSocketNonBlocking(sock->Get())) { LogPrintf("Error setting socket to non-blocking: %s\n", NetworkErrorString(WSAGetLastError())); return nullptr; } - return std::make_unique<Sock>(hSocket); + return sock; } std::function<std::unique_ptr<Sock>(const CService&)> CreateSock = CreateSockTCP; @@ -669,7 +675,7 @@ bool ConnectThroughProxy(const Proxy& proxy, const std::string& strDest, uint16_ return false; } } else { - if (!Socks5(strDest, port, 0, sock)) { + if (!Socks5(strDest, port, nullptr, sock)) { return false; } } @@ -678,7 +684,7 @@ bool ConnectThroughProxy(const Proxy& proxy, const std::string& strDest, uint16_ bool LookupSubNet(const std::string& subnet_str, CSubNet& subnet_out) { - if (!ValidAsCString(subnet_str)) { + if (!ContainsNoNUL(subnet_str)) { return false; } @@ -711,40 +717,21 @@ bool LookupSubNet(const std::string& subnet_str, CSubNet& subnet_out) return false; } -bool SetSocketNonBlocking(const SOCKET& hSocket, bool fNonBlocking) +bool SetSocketNonBlocking(const SOCKET& hSocket) { - if (fNonBlocking) { -#ifdef WIN32 - u_long nOne = 1; - if (ioctlsocket(hSocket, FIONBIO, &nOne) == SOCKET_ERROR) { -#else - int fFlags = fcntl(hSocket, F_GETFL, 0); - if (fcntl(hSocket, F_SETFL, fFlags | O_NONBLOCK) == SOCKET_ERROR) { -#endif - return false; - } - } else { #ifdef WIN32 - u_long nZero = 0; - if (ioctlsocket(hSocket, FIONBIO, &nZero) == SOCKET_ERROR) { + u_long nOne = 1; + if (ioctlsocket(hSocket, FIONBIO, &nOne) == SOCKET_ERROR) { #else - int fFlags = fcntl(hSocket, F_GETFL, 0); - if (fcntl(hSocket, F_SETFL, fFlags & ~O_NONBLOCK) == SOCKET_ERROR) { + int fFlags = fcntl(hSocket, F_GETFL, 0); + if (fcntl(hSocket, F_SETFL, fFlags | O_NONBLOCK) == SOCKET_ERROR) { #endif - return false; - } + return false; } return true; } -bool SetSocketNoDelay(const SOCKET& hSocket) -{ - int set = 1; - int rc = setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (const char*)&set, sizeof(int)); - return rc == 0; -} - void InterruptSocks5(bool interrupt) { interruptSocks5Recv = interrupt; diff --git a/src/netbase.h b/src/netbase.h index f9e3872c16..bf7522210d 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -221,10 +221,8 @@ bool ConnectSocketDirectly(const CService &addrConnect, const Sock& sock, int nT */ bool ConnectThroughProxy(const Proxy& proxy, const std::string& strDest, uint16_t port, const Sock& sock, int nTimeout, bool& outProxyConnectionFailed); -/** Disable or enable blocking-mode for a socket */ -bool SetSocketNonBlocking(const SOCKET& hSocket, bool fNonBlocking); -/** Set the TCP_NODELAY flag on a socket */ -bool SetSocketNoDelay(const SOCKET& hSocket); +/** Enable non-blocking mode for a socket */ +bool SetSocketNonBlocking(const SOCKET& hSocket); void InterruptSocks5(bool interrupt); /** diff --git a/src/netgroup.cpp b/src/netgroup.cpp new file mode 100644 index 0000000000..96b5e29684 --- /dev/null +++ b/src/netgroup.cpp @@ -0,0 +1,111 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <netgroup.h> + +#include <hash.h> +#include <util/asmap.h> + +uint256 NetGroupManager::GetAsmapChecksum() const +{ + if (!m_asmap.size()) return {}; + + return SerializeHash(m_asmap); +} + +std::vector<unsigned char> NetGroupManager::GetGroup(const CNetAddr& address) const +{ + std::vector<unsigned char> vchRet; + // If non-empty asmap is supplied and the address is IPv4/IPv6, + // return ASN to be used for bucketing. + uint32_t asn = GetMappedAS(address); + if (asn != 0) { // Either asmap was empty, or address has non-asmappable net class (e.g. TOR). + vchRet.push_back(NET_IPV6); // IPv4 and IPv6 with same ASN should be in the same bucket + for (int i = 0; i < 4; i++) { + vchRet.push_back((asn >> (8 * i)) & 0xFF); + } + return vchRet; + } + + vchRet.push_back(address.GetNetClass()); + int nStartByte{0}; + int nBits{0}; + + if (address.IsLocal()) { + // all local addresses belong to the same group + } else if (address.IsInternal()) { + // All internal-usage addresses get their own group. + // Skip over the INTERNAL_IN_IPV6_PREFIX returned by CAddress::GetAddrBytes(). + nStartByte = INTERNAL_IN_IPV6_PREFIX.size(); + nBits = ADDR_INTERNAL_SIZE * 8; + } else if (!address.IsRoutable()) { + // all other unroutable addresses belong to the same group + } else if (address.HasLinkedIPv4()) { + // IPv4 addresses (and mapped IPv4 addresses) use /16 groups + uint32_t ipv4 = address.GetLinkedIPv4(); + vchRet.push_back((ipv4 >> 24) & 0xFF); + vchRet.push_back((ipv4 >> 16) & 0xFF); + return vchRet; + } else if (address.IsTor() || address.IsI2P()) { + nBits = 4; + } else if (address.IsCJDNS()) { + // Treat in the same way as Tor and I2P because the address in all of + // them is "random" bytes (derived from a public key). However in CJDNS + // the first byte is a constant 0xfc, so the random bytes come after it. + // Thus skip the constant 8 bits at the start. + nBits = 12; + } else if (address.IsHeNet()) { + // for he.net, use /36 groups + nBits = 36; + } else { + // for the rest of the IPv6 network, use /32 groups + nBits = 32; + } + + // Push our address onto vchRet. + auto addr_bytes = address.GetAddrBytes(); + const size_t num_bytes = nBits / 8; + vchRet.insert(vchRet.end(), addr_bytes.begin() + nStartByte, addr_bytes.begin() + nStartByte + num_bytes); + nBits %= 8; + // ...for the last byte, push nBits and for the rest of the byte push 1's + if (nBits > 0) { + assert(num_bytes < addr_bytes.size()); + vchRet.push_back(addr_bytes[num_bytes + nStartByte] | ((1 << (8 - nBits)) - 1)); + } + + return vchRet; +} + +uint32_t NetGroupManager::GetMappedAS(const CNetAddr& address) const +{ + uint32_t net_class = address.GetNetClass(); + if (m_asmap.size() == 0 || (net_class != NET_IPV4 && net_class != NET_IPV6)) { + return 0; // Indicates not found, safe because AS0 is reserved per RFC7607. + } + std::vector<bool> ip_bits(128); + if (address.HasLinkedIPv4()) { + // For lookup, treat as if it was just an IPv4 address (IPV4_IN_IPV6_PREFIX + IPv4 bits) + for (int8_t byte_i = 0; byte_i < 12; ++byte_i) { + for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { + ip_bits[byte_i * 8 + bit_i] = (IPV4_IN_IPV6_PREFIX[byte_i] >> (7 - bit_i)) & 1; + } + } + uint32_t ipv4 = address.GetLinkedIPv4(); + for (int i = 0; i < 32; ++i) { + ip_bits[96 + i] = (ipv4 >> (31 - i)) & 1; + } + } else { + // Use all 128 bits of the IPv6 address otherwise + assert(address.IsIPv6()); + auto addr_bytes = address.GetAddrBytes(); + for (int8_t byte_i = 0; byte_i < 16; ++byte_i) { + uint8_t cur_byte = addr_bytes[byte_i]; + for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { + ip_bits[byte_i * 8 + bit_i] = (cur_byte >> (7 - bit_i)) & 1; + } + } + } + uint32_t mapped_as = Interpret(m_asmap, ip_bits); + return mapped_as; +} diff --git a/src/netgroup.h b/src/netgroup.h new file mode 100644 index 0000000000..2dd63ec66b --- /dev/null +++ b/src/netgroup.h @@ -0,0 +1,66 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NETGROUP_H +#define BITCOIN_NETGROUP_H + +#include <netaddress.h> +#include <uint256.h> + +#include <vector> + +/** + * Netgroup manager + */ +class NetGroupManager { +public: + explicit NetGroupManager(std::vector<bool> asmap) + : m_asmap{std::move(asmap)} + {} + + /** Get a checksum identifying the asmap being used. */ + uint256 GetAsmapChecksum() const; + + /** + * Get the canonical identifier of the network group for address. + * + * The groups are assigned in a way where it should be costly for an attacker to + * obtain addresses with many different group identifiers, even if it is cheap + * to obtain addresses with the same identifier. + * + * @note No two connections will be attempted to addresses with the same network + * group. + */ + std::vector<unsigned char> GetGroup(const CNetAddr& address) const; + + /** + * Get the autonomous system on the BGP path to address. + * + * The ip->AS mapping depends on how asmap is constructed. + */ + uint32_t GetMappedAS(const CNetAddr& address) const; + +private: + /** Compressed IP->ASN mapping, loaded from a file when a node starts. + * + * This mapping is then used for bucketing nodes in Addrman and for + * ensuring we connect to a diverse set of peers in Connman. The map is + * empty if no file was provided. + * + * If asmap is provided, nodes will be bucketed by AS they belong to, in + * order to make impossible for a node to connect to several nodes hosted + * in a single AS. This is done in response to Erebus attack, but also to + * generally diversify the connections every node creates, especially + * useful when a large fraction of nodes operate under a couple of cloud + * providers. + * + * If a new asmap is provided, the existing addrman records are + * re-bucketed. + * + * This is initialized in the constructor, const, and therefore is + * thread-safe. */ + const std::vector<bool> m_asmap; +}; + +#endif // BITCOIN_NETGROUP_H diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 8a99130fd0..17ab226a30 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -21,54 +21,95 @@ #include <util/system.h> #include <validation.h> +#include <unordered_map> + namespace node { std::atomic_bool fImporting(false); std::atomic_bool fReindex(false); -bool fHavePruned = false; bool fPruneMode = false; uint64_t nPruneTarget = 0; +bool CBlockIndexWorkComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const +{ + // First sort by most total work, ... + if (pa->nChainWork > pb->nChainWork) return false; + if (pa->nChainWork < pb->nChainWork) return true; + + // ... then by earliest time received, ... + if (pa->nSequenceId < pb->nSequenceId) return false; + if (pa->nSequenceId > pb->nSequenceId) return true; + + // Use pointer address as tie breaker (should only happen with blocks + // loaded from disk, as those all have id 0). + if (pa < pb) return false; + if (pa > pb) return true; + + // Identical blocks. + return false; +} + +bool CBlockIndexHeightOnlyComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const +{ + return pa->nHeight < pb->nHeight; +} + static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false); static FlatFileSeq BlockFileSeq(); static FlatFileSeq UndoFileSeq(); -CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) const +std::vector<CBlockIndex*> BlockManager::GetAllBlockIndices() +{ + AssertLockHeld(cs_main); + std::vector<CBlockIndex*> rv; + rv.reserve(m_block_index.size()); + for (auto& [_, block_index] : m_block_index) { + rv.push_back(&block_index); + } + return rv; +} + +CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) +{ + AssertLockHeld(cs_main); + BlockMap::iterator it = m_block_index.find(hash); + return it == m_block_index.end() ? nullptr : &it->second; +} + +const CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) const { AssertLockHeld(cs_main); BlockMap::const_iterator it = m_block_index.find(hash); - return it == m_block_index.end() ? nullptr : it->second; + return it == m_block_index.end() ? nullptr : &it->second; } -CBlockIndex* BlockManager::AddToBlockIndex(const CBlockHeader& block) +CBlockIndex* BlockManager::AddToBlockIndex(const CBlockHeader& block, CBlockIndex*& best_header) { AssertLockHeld(cs_main); - // Check for duplicate - uint256 hash = block.GetHash(); - BlockMap::iterator it = m_block_index.find(hash); - if (it != m_block_index.end()) { - return it->second; + auto [mi, inserted] = m_block_index.try_emplace(block.GetHash(), block); + if (!inserted) { + return &mi->second; } + CBlockIndex* pindexNew = &(*mi).second; - // Construct new block index object - CBlockIndex* pindexNew = new CBlockIndex(block); // We assign the sequence id to blocks only when the full data is available, // to avoid miners withholding blocks but broadcasting headers, to get a // competitive advantage. pindexNew->nSequenceId = 0; - BlockMap::iterator mi = m_block_index.insert(std::make_pair(hash, pindexNew)).first; + pindexNew->phashBlock = &((*mi).first); BlockMap::iterator miPrev = m_block_index.find(block.hashPrevBlock); if (miPrev != m_block_index.end()) { - pindexNew->pprev = (*miPrev).second; + pindexNew->pprev = &(*miPrev).second; pindexNew->nHeight = pindexNew->pprev->nHeight + 1; pindexNew->BuildSkip(); } pindexNew->nTimeMax = (pindexNew->pprev ? std::max(pindexNew->pprev->nTimeMax, pindexNew->nTime) : pindexNew->nTime); pindexNew->nChainWork = (pindexNew->pprev ? pindexNew->pprev->nChainWork : 0) + GetBlockProof(*pindexNew); pindexNew->RaiseValidity(BLOCK_VALID_TREE); - if (pindexBestHeader == nullptr || pindexBestHeader->nChainWork < pindexNew->nChainWork) - pindexBestHeader = pindexNew; + if (best_header == nullptr || best_header->nChainWork < pindexNew->nChainWork) { + best_header = pindexNew; + } m_dirty_blockindex.insert(pindexNew); @@ -80,8 +121,8 @@ void BlockManager::PruneOneBlockFile(const int fileNumber) AssertLockHeld(cs_main); LOCK(cs_LastBlockFile); - for (const auto& entry : m_block_index) { - CBlockIndex* pindex = entry.second; + for (auto& entry : m_block_index) { + CBlockIndex* pindex = &entry.second; if (pindex->nFile == fileNumber) { pindex->nStatus &= ~BLOCK_HAVE_DATA; pindex->nStatus &= ~BLOCK_HAVE_UNDO; @@ -191,6 +232,11 @@ void BlockManager::FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPr nLastBlockWeCanPrune, count); } +void BlockManager::UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) { + AssertLockHeld(::cs_main); + m_prune_locks[name] = lock_info; +} + CBlockIndex* BlockManager::InsertBlockIndex(const uint256& hash) { AssertLockHeld(cs_main); @@ -199,60 +245,27 @@ CBlockIndex* BlockManager::InsertBlockIndex(const uint256& hash) return nullptr; } - // Return existing - BlockMap::iterator mi = m_block_index.find(hash); - if (mi != m_block_index.end()) { - return (*mi).second; + const auto [mi, inserted]{m_block_index.try_emplace(hash)}; + CBlockIndex* pindex = &(*mi).second; + if (inserted) { + pindex->phashBlock = &((*mi).first); } - - // Create new - CBlockIndex* pindexNew = new CBlockIndex(); - mi = m_block_index.insert(std::make_pair(hash, pindexNew)).first; - pindexNew->phashBlock = &((*mi).first); - - return pindexNew; + return pindex; } -bool BlockManager::LoadBlockIndex( - const Consensus::Params& consensus_params, - ChainstateManager& chainman) +bool BlockManager::LoadBlockIndex(const Consensus::Params& consensus_params) { if (!m_block_tree_db->LoadBlockIndexGuts(consensus_params, [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); })) { return false; } // Calculate nChainWork - std::vector<std::pair<int, CBlockIndex*>> vSortedByHeight; - vSortedByHeight.reserve(m_block_index.size()); - for (const std::pair<const uint256, CBlockIndex*>& item : m_block_index) { - CBlockIndex* pindex = item.second; - vSortedByHeight.push_back(std::make_pair(pindex->nHeight, pindex)); - } - sort(vSortedByHeight.begin(), vSortedByHeight.end()); - - // Find start of assumed-valid region. - int first_assumed_valid_height = std::numeric_limits<int>::max(); - - for (const auto& [height, block] : vSortedByHeight) { - if (block->IsAssumedValid()) { - auto chainstates = chainman.GetAll(); - - // If we encounter an assumed-valid block index entry, ensure that we have - // one chainstate that tolerates assumed-valid entries and another that does - // not (i.e. the background validation chainstate), since assumed-valid - // entries should always be pending validation by a fully-validated chainstate. - auto any_chain = [&](auto fnc) { return std::any_of(chainstates.cbegin(), chainstates.cend(), fnc); }; - assert(any_chain([](auto chainstate) { return chainstate->reliesOnAssumedValid(); })); - assert(any_chain([](auto chainstate) { return !chainstate->reliesOnAssumedValid(); })); - - first_assumed_valid_height = height; - break; - } - } + std::vector<CBlockIndex*> vSortedByHeight{GetAllBlockIndices()}; + std::sort(vSortedByHeight.begin(), vSortedByHeight.end(), + CBlockIndexHeightOnlyComparator()); - for (const std::pair<int, CBlockIndex*>& item : vSortedByHeight) { + for (CBlockIndex* pindex : vSortedByHeight) { if (ShutdownRequested()) return false; - CBlockIndex* pindex = item.second; pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex); pindex->nTimeMax = (pindex->pprev ? std::max(pindex->pprev->nTimeMax, pindex->nTime) : pindex->nTime); @@ -276,69 +289,14 @@ bool BlockManager::LoadBlockIndex( pindex->nStatus |= BLOCK_FAILED_CHILD; m_dirty_blockindex.insert(pindex); } - if (pindex->IsAssumedValid() || - (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && - (pindex->HaveTxsDownloaded() || pindex->pprev == nullptr))) { - - // Fill each chainstate's block candidate set. Only add assumed-valid - // blocks to the tip candidate set if the chainstate is allowed to rely on - // assumed-valid blocks. - // - // If all setBlockIndexCandidates contained the assumed-valid blocks, the - // background chainstate's ActivateBestChain() call would add assumed-valid - // blocks to the chain (based on how FindMostWorkChain() works). Obviously - // we don't want this since the purpose of the background validation chain - // is to validate assued-valid blocks. - // - // Note: This is considering all blocks whose height is greater or equal to - // the first assumed-valid block to be assumed-valid blocks, and excluding - // them from the background chainstate's setBlockIndexCandidates set. This - // does mean that some blocks which are not technically assumed-valid - // (later blocks on a fork beginning before the first assumed-valid block) - // might not get added to the background chainstate, but this is ok, - // because they will still be attached to the active chainstate if they - // actually contain more work. - // - // Instead of this height-based approach, an earlier attempt was made at - // detecting "holistically" whether the block index under consideration - // relied on an assumed-valid ancestor, but this proved to be too slow to - // be practical. - for (CChainState* chainstate : chainman.GetAll()) { - if (chainstate->reliesOnAssumedValid() || - pindex->nHeight < first_assumed_valid_height) { - chainstate->setBlockIndexCandidates.insert(pindex); - } - } - } - if (pindex->nStatus & BLOCK_FAILED_MASK && (!chainman.m_best_invalid || pindex->nChainWork > chainman.m_best_invalid->nChainWork)) { - chainman.m_best_invalid = pindex; - } if (pindex->pprev) { pindex->BuildSkip(); } - if (pindex->IsValid(BLOCK_VALID_TREE) && (pindexBestHeader == nullptr || CBlockIndexWorkComparator()(pindexBestHeader, pindex))) - pindexBestHeader = pindex; } return true; } -void BlockManager::Unload() -{ - m_blocks_unlinked.clear(); - - for (const BlockMap::value_type& entry : m_block_index) { - delete entry.second; - } - - m_block_index.clear(); - - m_blockfile_info.clear(); - m_last_blockfile = 0; - m_dirty_blockindex.clear(); - m_dirty_fileinfo.clear(); -} - bool BlockManager::WriteBlockIndexDB() { AssertLockHeld(::cs_main); @@ -360,9 +318,9 @@ bool BlockManager::WriteBlockIndexDB() return true; } -bool BlockManager::LoadBlockIndexDB(ChainstateManager& chainman) +bool BlockManager::LoadBlockIndexDB() { - if (!LoadBlockIndex(::Params().GetConsensus(), chainman)) { + if (!LoadBlockIndex(::Params().GetConsensus())) { return false; } @@ -386,10 +344,9 @@ bool BlockManager::LoadBlockIndexDB(ChainstateManager& chainman) // Check presence of blk files LogPrintf("Checking all blk files are present...\n"); std::set<int> setBlkDataFiles; - for (const std::pair<const uint256, CBlockIndex*>& item : m_block_index) { - CBlockIndex* pindex = item.second; - if (pindex->nStatus & BLOCK_HAVE_DATA) { - setBlkDataFiles.insert(pindex->nFile); + for (const auto& [_, block_index] : m_block_index) { + if (block_index.nStatus & BLOCK_HAVE_DATA) { + setBlkDataFiles.insert(block_index.nFile); } } for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) { @@ -400,8 +357,8 @@ bool BlockManager::LoadBlockIndexDB(ChainstateManager& chainman) } // Check whether we have ever pruned block & undo files - m_block_tree_db->ReadFlag("prunedblockfiles", fHavePruned); - if (fHavePruned) { + m_block_tree_db->ReadFlag("prunedblockfiles", m_have_pruned); + if (m_have_pruned) { LogPrintf("LoadBlockIndexDB(): Block files have previously been pruned\n"); } @@ -413,13 +370,13 @@ bool BlockManager::LoadBlockIndexDB(ChainstateManager& chainman) return true; } -CBlockIndex* BlockManager::GetLastCheckpoint(const CCheckpointData& data) +const CBlockIndex* BlockManager::GetLastCheckpoint(const CCheckpointData& data) { const MapCheckpoints& checkpoints = data.mapCheckpoints; for (const MapCheckpoints::value_type& i : reverse_iterate(checkpoints)) { const uint256& hash = i.second; - CBlockIndex* pindex = LookupBlockIndex(hash); + const CBlockIndex* pindex = LookupBlockIndex(hash); if (pindex) { return pindex; } @@ -427,10 +384,20 @@ CBlockIndex* BlockManager::GetLastCheckpoint(const CCheckpointData& data) return nullptr; } -bool IsBlockPruned(const CBlockIndex* pblockindex) +bool BlockManager::IsBlockPruned(const CBlockIndex* pblockindex) +{ + AssertLockHeld(::cs_main); + return (m_have_pruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0); +} + +const CBlockIndex* BlockManager::GetFirstStoredBlock(const CBlockIndex& start_block) { AssertLockHeld(::cs_main); - return (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0); + const CBlockIndex* last_block = &start_block; + while (last_block->pprev && (last_block->pprev->nStatus & BLOCK_HAVE_DATA)) { + last_block = last_block->pprev; + } + return last_block; } // If we're using -prune with -reindex, then delete block files that will be ignored by the diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 42e46797d2..488713dbd8 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -5,13 +5,16 @@ #ifndef BITCOIN_NODE_BLOCKSTORAGE_H #define BITCOIN_NODE_BLOCKSTORAGE_H +#include <attributes.h> +#include <chain.h> #include <fs.h> -#include <protocol.h> // For CMessageHeader::MessageStartChars +#include <protocol.h> #include <sync.h> #include <txdb.h> #include <atomic> #include <cstdint> +#include <unordered_map> #include <vector> extern RecursiveMutex cs_main; @@ -20,7 +23,6 @@ class ArgsManager; class BlockValidationState; class CBlock; class CBlockFileInfo; -class CBlockIndex; class CBlockUndo; class CChain; class CChainParams; @@ -45,19 +47,30 @@ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB extern std::atomic_bool fImporting; extern std::atomic_bool fReindex; /** Pruning-related variables and constants */ -/** True if any block files have ever been pruned. */ -extern bool fHavePruned; /** True if we're running in -prune mode. */ extern bool fPruneMode; /** Number of MiB of block files that we're trying to stay below. */ extern uint64_t nPruneTarget; -typedef std::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; +// Because validation code takes pointers to the map's CBlockIndex objects, if +// we ever switch to another associative container, we need to either use a +// container that has stable addressing (true of all std associative +// containers), or make the key a `std::unique_ptr<CBlockIndex>` +using BlockMap = std::unordered_map<uint256, CBlockIndex, BlockHasher>; struct CBlockIndexWorkComparator { bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const; }; +struct CBlockIndexHeightOnlyComparator { + /* Only compares the height of two block indices, doesn't try to tie-break */ + bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const; +}; + +struct PruneLockInfo { + int height_first{std::numeric_limits<int>::max()}; //! Height of earliest block that should be kept and not pruned +}; + /** * Maintains a tree of blocks (stored in `m_block_index`) which is consulted * to determine where the most-work tip is. @@ -71,6 +84,13 @@ class BlockManager friend ChainstateManager; private: + /** + * Load the blocktree off disk and into memory. Populate certain metadata + * per index entry (nStatus, nChainWork, nTimeMax, etc.) as well as peripheral + * collections like m_dirty_blockindex. + */ + bool LoadBlockIndex(const Consensus::Params& consensus_params) + EXCLUSIVE_LOCKS_REQUIRED(cs_main); void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false); void FlushUndoFile(int block_file, bool finalize = false); bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown); @@ -111,9 +131,19 @@ private: /** Dirty block file entries. */ std::set<int> m_dirty_fileinfo; + /** + * Map from external index name to oldest block that must not be pruned. + * + * @note Internally, only blocks at height (height_first - PRUNE_LOCK_BUFFER - 1) and + * below will be pruned, but callers should avoid assuming any particular buffer size. + */ + std::unordered_map<std::string, PruneLockInfo> m_prune_locks GUARDED_BY(::cs_main); + public: BlockMap m_block_index GUARDED_BY(cs_main); + std::vector<CBlockIndex*> GetAllBlockIndices() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + /** * All pairs A->B, where A (or one of its ancestors) misses transactions, but B has transactions. * Pruned nodes may have entries where B is missing data. @@ -123,28 +153,17 @@ public: std::unique_ptr<CBlockTreeDB> m_block_tree_db GUARDED_BY(::cs_main); bool WriteBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - bool LoadBlockIndexDB(ChainstateManager& chainman) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + bool LoadBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - /** - * Load the blocktree off disk and into memory. Populate certain metadata - * per index entry (nStatus, nChainWork, nTimeMax, etc.) as well as peripheral - * collections like m_dirty_blockindex. - */ - bool LoadBlockIndex( - const Consensus::Params& consensus_params, - ChainstateManager& chainman) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - - /** Clear all data members. */ - void Unload() EXCLUSIVE_LOCKS_REQUIRED(cs_main); - - CBlockIndex* AddToBlockIndex(const CBlockHeader& block) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + CBlockIndex* AddToBlockIndex(const CBlockHeader& block, CBlockIndex*& best_header) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Create a new block index entry for a given block hash */ CBlockIndex* InsertBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); //! Mark one block file as pruned (modify associated database entries) void PruneOneBlockFile(const int fileNumber) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - CBlockIndex* LookupBlockIndex(const uint256& hash) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); + CBlockIndex* LookupBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + const CBlockIndex* LookupBlockIndex(const uint256& hash) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Get block file info entry for one block file */ CBlockFileInfo* GetBlockFileInfo(size_t n); @@ -158,16 +177,20 @@ public: uint64_t CalculateCurrentUsage(); //! Returns last CBlockIndex* that is a checkpoint - CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + const CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - ~BlockManager() - { - Unload(); - } -}; + //! Find the first block that is not pruned + const CBlockIndex* GetFirstStoredBlock(const CBlockIndex& start_block LIFETIMEBOUND) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); -//! Check whether the block associated with this index entry is pruned or not. -bool IsBlockPruned(const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + /** True if any block files have ever been pruned. */ + bool m_have_pruned = false; + + //! Check whether the block associated with this index entry is pruned or not. + bool IsBlockPruned(const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + //! Create or update a prune lock identified by its name + void UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); +}; void CleanupBlockRevFiles(); diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index d03b9dcac6..99615dea69 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -32,8 +32,6 @@ std::optional<ChainstateLoadingError> LoadChainstate(bool fReset, chainman.m_total_coinstip_cache = nCoinCacheUsage; chainman.m_total_coinsdb_cache = nCoinDBCache; - UnloadBlockIndex(mempool, chainman); - auto& pblocktree{chainman.m_blockman.m_block_tree_db}; // new CBlockTreeDB tries to delete the existing file, which // fails if it's still open from the previous loop. Close it first: @@ -49,7 +47,7 @@ std::optional<ChainstateLoadingError> LoadChainstate(bool fReset, if (shutdown_requested && shutdown_requested()) return ChainstateLoadingError::SHUTDOWN_PROBED; - // LoadBlockIndex will load fHavePruned if we've ever removed a + // LoadBlockIndex will load m_have_pruned if we've ever removed a // block file from disk. // Note that it also sets fReindex based on the disk flag! // From here on out fReindex and fReset mean something different! @@ -65,7 +63,7 @@ std::optional<ChainstateLoadingError> LoadChainstate(bool fReset, // Check for changed -prune state. What we are concerned about is a user who has pruned blocks // in the past, but is now trying to run unpruned. - if (fHavePruned && !fPruneMode) { + if (chainman.m_blockman.m_have_pruned && !fPruneMode) { return ChainstateLoadingError::ERROR_PRUNED_NEEDS_REINDEX; } @@ -82,17 +80,17 @@ std::optional<ChainstateLoadingError> LoadChainstate(bool fReset, for (CChainState* chainstate : chainman.GetAll()) { chainstate->InitCoinsDB( - /* cache_size_bytes */ nCoinDBCache, - /* in_memory */ coins_db_in_memory, - /* should_wipe */ fReset || fReindexChainState); + /*cache_size_bytes=*/nCoinDBCache, + /*in_memory=*/coins_db_in_memory, + /*should_wipe=*/fReset || fReindexChainState); if (coins_error_cb) { chainstate->CoinsErrorCatcher().AddReadErrCallback(coins_error_cb); } - // If necessary, upgrade from older database format. + // Refuse to load unsupported database format. // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate - if (!chainstate->CoinsDB().Upgrade()) { + if (chainstate->CoinsDB().NeedsUpgrade()) { return ChainstateLoadingError::ERROR_CHAINSTATE_UPGRADE_FAILED; } diff --git a/src/node/context.cpp b/src/node/context.cpp index 893c32f1bc..0b31c10f44 100644 --- a/src/node/context.cpp +++ b/src/node/context.cpp @@ -9,6 +9,7 @@ #include <interfaces/chain.h> #include <net.h> #include <net_processing.h> +#include <netgroup.h> #include <policy/fees.h> #include <scheduler.h> #include <txmempool.h> diff --git a/src/node/context.h b/src/node/context.h index 644c997531..91ba456219 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -18,6 +18,7 @@ class CConnman; class CScheduler; class CTxMemPool; class ChainstateManager; +class NetGroupManager; class PeerManager; namespace interfaces { class Chain; @@ -43,6 +44,7 @@ struct NodeContext { std::unique_ptr<AddrMan> addrman; std::unique_ptr<CConnman> connman; std::unique_ptr<CTxMemPool> mempool; + std::unique_ptr<const NetGroupManager> netgroupman; std::unique_ptr<CBlockPolicyEstimator> fee_estimator; std::unique_ptr<PeerManager> peerman; std::unique_ptr<ChainstateManager> chainman; diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index cb063ae9f8..954bd1c31d 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -90,7 +90,7 @@ public: uint32_t getLogCategories() override { return LogInstance().GetCategoryMask(); } bool baseInitialize() override { - return AppInitBasicSetup(gArgs) && AppInitParameterInteraction(gArgs) && AppInitSanityChecks() && + return AppInitBasicSetup(gArgs) && AppInitParameterInteraction(gArgs, /*use_syscall_sandbox=*/false) && AppInitSanityChecks() && AppInitLockDataDirectory() && AppInitInterfaces(*m_context); } bool appInitMain(interfaces::BlockAndHeaderTipInfo* tip_info) override @@ -212,9 +212,10 @@ public: bool getHeaderTip(int& height, int64_t& block_time) override { LOCK(::cs_main); - if (::pindexBestHeader) { - height = ::pindexBestHeader->nHeight; - block_time = ::pindexBestHeader->GetBlockTime(); + auto best_header = chainman().m_best_header; + if (best_header) { + height = best_header->nHeight; + block_time = best_header->GetBlockTime(); return true; } return false; @@ -490,7 +491,7 @@ public: { LOCK(cs_main); const CChainState& active = Assert(m_node.chainman)->ActiveChainstate(); - if (CBlockIndex* fork = active.FindForkInGlobalIndex(locator)) { + if (const CBlockIndex* fork = active.FindForkInGlobalIndex(locator)) { return fork->nHeight; } return std::nullopt; @@ -557,7 +558,7 @@ public: // used to limit the range, and passing min_height that's too low or // max_height that's too high will not crash or change the result. LOCK(::cs_main); - if (CBlockIndex* block = chainman().m_blockman.LookupBlockIndex(block_hash)) { + if (const CBlockIndex* block = chainman().m_blockman.LookupBlockIndex(block_hash)) { if (max_height && block->nHeight >= *max_height) block = block->GetAncestor(*max_height); for (; block->nStatus & BLOCK_HAVE_DATA; block = block->pprev) { // Check pprev to not segfault if min_height is too low @@ -590,7 +591,7 @@ public: bool relay, std::string& err_string) override { - const TransactionError err = BroadcastTransaction(m_node, tx, err_string, max_tx_fee, relay, /*wait_callback*/ false); + const TransactionError err = BroadcastTransaction(m_node, tx, err_string, max_tx_fee, relay, /*wait_callback=*/false); // Chain clients only care about failures to accept the tx to the mempool. Disregard non-mempool related failures. // Note: this will need to be updated if BroadcastTransactions() is updated to return other non-mempool failures // that Chain clients do not need to know about. @@ -644,7 +645,7 @@ public: bool havePruned() override { LOCK(cs_main); - return node::fHavePruned; + return m_node.chainman->m_blockman.m_have_pruned; } bool isReadyToBroadcast() override { return !node::fImporting && !node::fReindex && !isInitialBlockDownload(); } bool isInitialBlockDownload() override { diff --git a/src/node/miner.cpp b/src/node/miner.cpp index 7fe10ecabc..be5d58527b 100644 --- a/src/node/miner.cpp +++ b/src/node/miner.cpp @@ -50,7 +50,7 @@ void RegenerateCommitments(CBlock& block, ChainstateManager& chainman) tx.vout.erase(tx.vout.begin() + GetWitnessCommitmentIndex(block)); block.vtx.at(0) = MakeTransactionRef(tx); - CBlockIndex* prev_block = WITH_LOCK(::cs_main, return chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock)); + const CBlockIndex* prev_block = WITH_LOCK(::cs_main, return chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock)); GenerateCoinbaseCommitment(block, prev_block, Params().GetConsensus()); block.hashMerkleRoot = BlockMerkleRoot(block); @@ -97,7 +97,6 @@ void BlockAssembler::resetBlock() // Reserve space for coinbase tx nBlockWeight = 4000; nBlockSigOpsCost = 400; - fIncludeWitness = false; // These counters do not include coinbase tx nBlockTx = 0; @@ -137,17 +136,6 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc pblock->nTime = GetAdjustedTime(); m_lock_time_cutoff = pindexPrev->GetMedianTimePast(); - // Decide whether to include witness transactions - // This is only needed in case the witness softfork activation is reverted - // (which would require a very deep reorganization). - // Note that the mempool would accept transactions with witness data before - // the deployment is active, but we would only ever mine blocks after activation - // unless there is a massive block reorganization with the witness softfork - // not activated. - // TODO: replace this with a call to main to assess validity of a mempool - // transaction (which in most cases can be a no-op). - fIncludeWitness = DeploymentActiveAfter(pindexPrev, chainparams.GetConsensus(), Consensus::DEPLOYMENT_SEGWIT); - int nPackagesSelected = 0; int nDescendantsUpdated = 0; addPackageTxs(nPackagesSelected, nDescendantsUpdated); @@ -215,17 +203,12 @@ bool BlockAssembler::TestPackage(uint64_t packageSize, int64_t packageSigOpsCost // Perform transaction-level checks before adding to block: // - transaction finality (locktime) -// - premature witness (in case segwit transactions are added to mempool before -// segwit activation) bool BlockAssembler::TestPackageTransactions(const CTxMemPool::setEntries& package) const { for (CTxMemPool::txiter it : package) { if (!IsFinalTx(it->GetTx(), nHeight, m_lock_time_cutoff)) { return false; } - if (!fIncludeWitness && it->GetTx().HasWitness()) { - return false; - } } return true; } @@ -447,22 +430,4 @@ void BlockAssembler::addPackageTxs(int& nPackagesSelected, int& nDescendantsUpda nDescendantsUpdated += UpdatePackagesForAdded(ancestors, mapModifiedTx); } } - -void IncrementExtraNonce(CBlock* pblock, const CBlockIndex* pindexPrev, unsigned int& nExtraNonce) -{ - // Update nExtraNonce - static uint256 hashPrevBlock; - if (hashPrevBlock != pblock->hashPrevBlock) { - nExtraNonce = 0; - hashPrevBlock = pblock->hashPrevBlock; - } - ++nExtraNonce; - unsigned int nHeight = pindexPrev->nHeight + 1; // Height first in coinbase required for block.version=2 - CMutableTransaction txCoinbase(*pblock->vtx[0]); - txCoinbase.vin[0].scriptSig = (CScript() << nHeight << CScriptNum(nExtraNonce)); - assert(txCoinbase.vin[0].scriptSig.size() <= 100); - - pblock->vtx[0] = MakeTransactionRef(std::move(txCoinbase)); - pblock->hashMerkleRoot = BlockMerkleRoot(*pblock); -} } // namespace node diff --git a/src/node/miner.h b/src/node/miner.h index c96da874a7..678df815c0 100644 --- a/src/node/miner.h +++ b/src/node/miner.h @@ -45,7 +45,7 @@ struct CTxMemPoolModifiedEntry { nSigOpCostWithAncestors = entry->GetSigOpCostWithAncestors(); } - int64_t GetModifiedFee() const { return iter->GetModifiedFee(); } + CAmount GetModifiedFee() const { return iter->GetModifiedFee(); } uint64_t GetSizeWithAncestors() const { return nSizeWithAncestors; } CAmount GetModFeesWithAncestors() const { return nModFeesWithAncestors; } size_t GetTxSize() const { return iter->GetTxSize(); } @@ -116,7 +116,7 @@ struct update_for_parent_inclusion void operator() (CTxMemPoolModifiedEntry &e) { - e.nModFeesWithAncestors -= iter->GetFee(); + e.nModFeesWithAncestors -= iter->GetModifiedFee(); e.nSizeWithAncestors -= iter->GetTxSize(); e.nSigOpCostWithAncestors -= iter->GetSigOpCost(); } @@ -132,7 +132,6 @@ private: std::unique_ptr<CBlockTemplate> pblocktemplate; // Configuration parameters for the block size - bool fIncludeWitness; unsigned int nBlockMaxWeight; CFeeRate blockMinFeeRate; @@ -201,8 +200,6 @@ private: int UpdatePackagesForAdded(const CTxMemPool::setEntries& alreadyAdded, indexed_modified_transaction_set& mapModifiedTx) EXCLUSIVE_LOCKS_REQUIRED(m_mempool.cs); }; -/** Modify the extranonce in a block */ -void IncrementExtraNonce(CBlock* pblock, const CBlockIndex* pindexPrev, unsigned int& nExtraNonce); int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev); /** Update an old GenerateCoinbaseCommitment from CreateNewBlock after the block txs have changed */ diff --git a/src/prevector.h b/src/prevector.h index aa20efaaa7..830b31e315 100644 --- a/src/prevector.h +++ b/src/prevector.h @@ -458,7 +458,8 @@ public: return *item_ptr(size() - 1); } - void swap(prevector<N, T, Size, Diff>& other) { + void swap(prevector<N, T, Size, Diff>& other) noexcept + { std::swap(_union, other._union); std::swap(_size, other._size); } diff --git a/src/psbt.cpp b/src/psbt.cpp index c8c73e130b..c1c8a385cc 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -234,7 +234,7 @@ void UpdatePSBTOutput(const SigningProvider& provider, PartiallySignedTransactio // Construct a would-be spend of this output, to update sigdata with. // Note that ProduceSignature is used to fill in metadata (not actual signatures), // so provider does not need to provide any private keys (it can be a HidingSigningProvider). - MutableTransactionSignatureCreator creator(&tx, /*input_idx=*/0, out.nValue, SIGHASH_ALL); + MutableTransactionSignatureCreator creator(tx, /*input_idx=*/0, out.nValue, SIGHASH_ALL); ProduceSignature(provider, creator, out.scriptPubKey, sigdata); // Put redeem_script, witness_script, key paths, into PSBTOutput. @@ -301,7 +301,7 @@ bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& if (txdata == nullptr) { sig_complete = ProduceSignature(provider, DUMMY_SIGNATURE_CREATOR, utxo.scriptPubKey, sigdata); } else { - MutableTransactionSignatureCreator creator(&tx, index, utxo.nValue, txdata, sighash); + MutableTransactionSignatureCreator creator(tx, index, utxo.nValue, txdata, sighash); sig_complete = ProduceSignature(provider, creator, utxo.scriptPubKey, sigdata); } // Verify that a witness signature was produced in case one was required. @@ -388,18 +388,17 @@ std::string PSBTRoleName(PSBTRole role) { bool DecodeBase64PSBT(PartiallySignedTransaction& psbt, const std::string& base64_tx, std::string& error) { - bool invalid; - std::string tx_data = DecodeBase64(base64_tx, &invalid); - if (invalid) { + auto tx_data = DecodeBase64(base64_tx); + if (!tx_data) { error = "invalid base64"; return false; } - return DecodeRawPSBT(psbt, tx_data, error); + return DecodeRawPSBT(psbt, MakeByteSpan(*tx_data), error); } -bool DecodeRawPSBT(PartiallySignedTransaction& psbt, const std::string& tx_data, std::string& error) +bool DecodeRawPSBT(PartiallySignedTransaction& psbt, Span<const std::byte> tx_data, std::string& error) { - CDataStream ss_data(MakeByteSpan(tx_data), SER_NETWORK, PROTOCOL_VERSION); + CDataStream ss_data(tx_data, SER_NETWORK, PROTOCOL_VERSION); try { ss_data >> psbt; if (!ss_data.empty()) { diff --git a/src/psbt.h b/src/psbt.h index f0ceb02481..8a9cbd33d2 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -988,6 +988,6 @@ bool FinalizeAndExtractPSBT(PartiallySignedTransaction& psbtx, CMutableTransacti //! Decode a base64ed PSBT into a PartiallySignedTransaction [[nodiscard]] bool DecodeBase64PSBT(PartiallySignedTransaction& decoded_psbt, const std::string& base64_psbt, std::string& error); //! Decode a raw (binary blob) PSBT into a PartiallySignedTransaction -[[nodiscard]] bool DecodeRawPSBT(PartiallySignedTransaction& decoded_psbt, const std::string& raw_psbt, std::string& error); +[[nodiscard]] bool DecodeRawPSBT(PartiallySignedTransaction& decoded_psbt, Span<const std::byte> raw_psbt, std::string& error); #endif // BITCOIN_PSBT_H diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index d6f80e1558..8cac28400f 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -95,6 +95,8 @@ static void RegisterMetaTypes() qRegisterMetaType<std::function<void()>>("std::function<void()>"); qRegisterMetaType<QMessageBox::Icon>("QMessageBox::Icon"); qRegisterMetaType<interfaces::BlockAndHeaderTipInfo>("interfaces::BlockAndHeaderTipInfo"); + + qRegisterMetaTypeStreamOperators<BitcoinUnit>("BitcoinUnit"); } static QString GetLangTerritory() @@ -587,7 +589,7 @@ int GuiMain(int argc, char* argv[]) return EXIT_FAILURE; } #ifdef ENABLE_WALLET - // Parse URIs on command line -- this can affect Params() + // Parse URIs on command line PaymentServer::ipcParseCommandLine(argc, argv); #endif diff --git a/src/qt/bitcoinamountfield.cpp b/src/qt/bitcoinamountfield.cpp index a257e250e0..c92aecd095 100644 --- a/src/qt/bitcoinamountfield.cpp +++ b/src/qt/bitcoinamountfield.cpp @@ -14,6 +14,9 @@ #include <QHBoxLayout> #include <QKeyEvent> #include <QLineEdit> +#include <QVariant> + +#include <cassert> /** QSpinBox that uses fixed-point numbers internally and uses our own * formatting/parsing functions. @@ -96,7 +99,7 @@ public: setValue(val); } - void setDisplayUnit(int unit) + void setDisplayUnit(BitcoinUnit unit) { bool valid = false; CAmount val = value(&valid); @@ -122,7 +125,7 @@ public: const QFontMetrics fm(fontMetrics()); int h = lineEdit()->minimumSizeHint().height(); - int w = GUIUtil::TextWidth(fm, BitcoinUnits::format(BitcoinUnits::BTC, BitcoinUnits::maxMoney(), false, BitcoinUnits::SeparatorStyle::ALWAYS)); + int w = GUIUtil::TextWidth(fm, BitcoinUnits::format(BitcoinUnit::BTC, BitcoinUnits::maxMoney(), false, BitcoinUnits::SeparatorStyle::ALWAYS)); w += 2; // cursor blinking space QStyleOptionSpinBox opt; @@ -141,14 +144,13 @@ public: opt.rect = rect(); - cachedMinimumSizeHint = style()->sizeFromContents(QStyle::CT_SpinBox, &opt, hint, this) - .expandedTo(QApplication::globalStrut()); + cachedMinimumSizeHint = style()->sizeFromContents(QStyle::CT_SpinBox, &opt, hint, this); } return cachedMinimumSizeHint; } private: - int currentUnit{BitcoinUnits::BTC}; + BitcoinUnit currentUnit{BitcoinUnit::BTC}; CAmount singleStep{CAmount(100000)}; // satoshis mutable QSize cachedMinimumSizeHint; bool m_allow_empty{true}; @@ -326,14 +328,14 @@ void BitcoinAmountField::unitChanged(int idx) unit->setToolTip(unit->itemData(idx, Qt::ToolTipRole).toString()); // Determine new unit ID - int newUnit = unit->itemData(idx, BitcoinUnits::UnitRole).toInt(); - - amount->setDisplayUnit(newUnit); + QVariant new_unit = unit->currentData(BitcoinUnits::UnitRole); + assert(new_unit.isValid()); + amount->setDisplayUnit(new_unit.value<BitcoinUnit>()); } -void BitcoinAmountField::setDisplayUnit(int newUnit) +void BitcoinAmountField::setDisplayUnit(BitcoinUnit new_unit) { - unit->setValue(newUnit); + unit->setValue(QVariant::fromValue(new_unit)); } void BitcoinAmountField::setSingleStep(const CAmount& step) diff --git a/src/qt/bitcoinamountfield.h b/src/qt/bitcoinamountfield.h index 366a6fc4b5..a40cd38332 100644 --- a/src/qt/bitcoinamountfield.h +++ b/src/qt/bitcoinamountfield.h @@ -6,6 +6,7 @@ #define BITCOIN_QT_BITCOINAMOUNTFIELD_H #include <consensus/amount.h> +#include <qt/bitcoinunits.h> #include <QWidget> @@ -52,7 +53,7 @@ public: bool validate(); /** Change unit used to display amount. */ - void setDisplayUnit(int unit); + void setDisplayUnit(BitcoinUnit new_unit); /** Make field empty and ready for new input. */ void clear(); diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 7c22880dd1..81b0e711b2 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -41,11 +41,13 @@ #include <validation.h> #include <QAction> +#include <QActionGroup> #include <QApplication> #include <QComboBox> #include <QCursor> #include <QDateTime> #include <QDragEnterEvent> +#include <QKeySequence> #include <QListWidget> #include <QMenu> #include <QMenuBar> @@ -251,28 +253,28 @@ void BitcoinGUI::createActions() overviewAction->setStatusTip(tr("Show general overview of wallet")); overviewAction->setToolTip(overviewAction->statusTip()); overviewAction->setCheckable(true); - overviewAction->setShortcut(QKeySequence(Qt::ALT + Qt::Key_1)); + overviewAction->setShortcut(QKeySequence(QStringLiteral("Alt+1"))); tabGroup->addAction(overviewAction); sendCoinsAction = new QAction(platformStyle->SingleColorIcon(":/icons/send"), tr("&Send"), this); sendCoinsAction->setStatusTip(tr("Send coins to a Bitcoin address")); sendCoinsAction->setToolTip(sendCoinsAction->statusTip()); sendCoinsAction->setCheckable(true); - sendCoinsAction->setShortcut(QKeySequence(Qt::ALT + Qt::Key_2)); + sendCoinsAction->setShortcut(QKeySequence(QStringLiteral("Alt+2"))); tabGroup->addAction(sendCoinsAction); receiveCoinsAction = new QAction(platformStyle->SingleColorIcon(":/icons/receiving_addresses"), tr("&Receive"), this); receiveCoinsAction->setStatusTip(tr("Request payments (generates QR codes and bitcoin: URIs)")); receiveCoinsAction->setToolTip(receiveCoinsAction->statusTip()); receiveCoinsAction->setCheckable(true); - receiveCoinsAction->setShortcut(QKeySequence(Qt::ALT + Qt::Key_3)); + receiveCoinsAction->setShortcut(QKeySequence(QStringLiteral("Alt+3"))); tabGroup->addAction(receiveCoinsAction); historyAction = new QAction(platformStyle->SingleColorIcon(":/icons/history"), tr("&Transactions"), this); historyAction->setStatusTip(tr("Browse transaction history")); historyAction->setToolTip(historyAction->statusTip()); historyAction->setCheckable(true); - historyAction->setShortcut(QKeySequence(Qt::ALT + Qt::Key_4)); + historyAction->setShortcut(QKeySequence(QStringLiteral("Alt+4"))); tabGroup->addAction(historyAction); #ifdef ENABLE_WALLET @@ -290,7 +292,7 @@ void BitcoinGUI::createActions() quitAction = new QAction(tr("E&xit"), this); quitAction->setStatusTip(tr("Quit application")); - quitAction->setShortcut(QKeySequence(Qt::CTRL + Qt::Key_Q)); + quitAction->setShortcut(QKeySequence(tr("Ctrl+Q"))); quitAction->setMenuRole(QAction::QuitRole); aboutAction = new QAction(tr("&About %1").arg(PACKAGE_NAME), this); aboutAction->setStatusTip(tr("Show information about %1").arg(PACKAGE_NAME)); @@ -354,7 +356,7 @@ void BitcoinGUI::createActions() showHelpMessageAction->setStatusTip(tr("Show the %1 help message to get a list with possible Bitcoin command-line options").arg(PACKAGE_NAME)); m_mask_values_action = new QAction(tr("&Mask values"), this); - m_mask_values_action->setShortcut(QKeySequence(Qt::CTRL + Qt::SHIFT + Qt::Key_M)); + m_mask_values_action->setShortcut(QKeySequence(Qt::CTRL | Qt::SHIFT | Qt::Key_M)); m_mask_values_action->setStatusTip(tr("Mask the values in the Overview tab")); m_mask_values_action->setCheckable(true); @@ -425,8 +427,8 @@ void BitcoinGUI::createActions() } #endif // ENABLE_WALLET - connect(new QShortcut(QKeySequence(Qt::CTRL + Qt::SHIFT + Qt::Key_C), this), &QShortcut::activated, this, &BitcoinGUI::showDebugWindowActivateConsole); - connect(new QShortcut(QKeySequence(Qt::CTRL + Qt::SHIFT + Qt::Key_D), this), &QShortcut::activated, this, &BitcoinGUI::showDebugWindow); + connect(new QShortcut(QKeySequence(Qt::CTRL | Qt::SHIFT | Qt::Key_C), this), &QShortcut::activated, this, &BitcoinGUI::showDebugWindowActivateConsole); + connect(new QShortcut(QKeySequence(Qt::CTRL | Qt::SHIFT | Qt::Key_D), this), &QShortcut::activated, this, &BitcoinGUI::showDebugWindow); } void BitcoinGUI::createMenuBar() @@ -472,7 +474,7 @@ void BitcoinGUI::createMenuBar() QMenu* window_menu = appMenuBar->addMenu(tr("&Window")); QAction* minimize_action = window_menu->addAction(tr("&Minimize")); - minimize_action->setShortcut(QKeySequence(Qt::CTRL + Qt::Key_M)); + minimize_action->setShortcut(QKeySequence(tr("Ctrl+M"))); connect(minimize_action, &QAction::triggered, [] { QApplication::activeWindow()->showMinimized(); }); @@ -852,7 +854,7 @@ void BitcoinGUI::aboutClicked() if(!clientModel) return; - auto dlg = new HelpMessageDialog(this, /* about */ true); + auto dlg = new HelpMessageDialog(this, /*about=*/true); GUIUtil::ShowModalDialogAsynchronously(dlg); } @@ -1243,7 +1245,7 @@ void BitcoinGUI::showEvent(QShowEvent *event) } #ifdef ENABLE_WALLET -void BitcoinGUI::incomingTransaction(const QString& date, int unit, const CAmount& amount, const QString& type, const QString& address, const QString& label, const QString& walletName) +void BitcoinGUI::incomingTransaction(const QString& date, BitcoinUnit unit, const CAmount& amount, const QString& type, const QString& address, const QString& label, const QString& walletName) { // On new transaction, make an info balloon QString msg = tr("Date: %1\n").arg(date) + @@ -1494,11 +1496,10 @@ UnitDisplayStatusBarControl::UnitDisplayStatusBarControl(const PlatformStyle *pl { createContextMenu(); setToolTip(tr("Unit to show amounts in. Click to select another unit.")); - QList<BitcoinUnits::Unit> units = BitcoinUnits::availableUnits(); + QList<BitcoinUnit> units = BitcoinUnits::availableUnits(); int max_width = 0; const QFontMetrics fm(font()); - for (const BitcoinUnits::Unit unit : units) - { + for (const BitcoinUnit unit : units) { max_width = qMax(max_width, GUIUtil::TextWidth(fm, BitcoinUnits::longName(unit))); } setMinimumSize(max_width, 0); @@ -1528,8 +1529,8 @@ void UnitDisplayStatusBarControl::changeEvent(QEvent* e) void UnitDisplayStatusBarControl::createContextMenu() { menu = new QMenu(this); - for (const BitcoinUnits::Unit u : BitcoinUnits::availableUnits()) { - menu->addAction(BitcoinUnits::longName(u))->setData(QVariant(u)); + for (const BitcoinUnit u : BitcoinUnits::availableUnits()) { + menu->addAction(BitcoinUnits::longName(u))->setData(QVariant::fromValue(u)); } connect(menu, &QMenu::triggered, this, &UnitDisplayStatusBarControl::onMenuSelection); } @@ -1550,7 +1551,7 @@ void UnitDisplayStatusBarControl::setOptionsModel(OptionsModel *_optionsModel) } /** When Display Units are changed on OptionsModel it will refresh the display text of the control on the status bar */ -void UnitDisplayStatusBarControl::updateDisplayUnit(int newUnits) +void UnitDisplayStatusBarControl::updateDisplayUnit(BitcoinUnit newUnits) { setText(BitcoinUnits::longName(newUnits)); } diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index 0ae5f7331e..5d9a978695 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -9,6 +9,7 @@ #include <config/bitcoin-config.h> #endif +#include <qt/bitcoinunits.h> #include <qt/guiutil.h> #include <qt/optionsdialog.h> @@ -260,7 +261,7 @@ public Q_SLOTS: bool handlePaymentRequest(const SendCoinsRecipient& recipient); /** Show incoming transaction notification for new transactions. */ - void incomingTransaction(const QString& date, int unit, const CAmount& amount, const QString& type, const QString& address, const QString& label, const QString& walletName); + void incomingTransaction(const QString& date, BitcoinUnit unit, const CAmount& amount, const QString& type, const QString& address, const QString& label, const QString& walletName); #endif // ENABLE_WALLET private: @@ -303,7 +304,7 @@ public Q_SLOTS: /** Show window if hidden, unminimize when minimized, rise when obscured or show if hidden and fToggleHidden is true */ void showNormalIfMinimized() { showNormalIfMinimized(false); } void showNormalIfMinimized(bool fToggleHidden); - /** Simply calls showNormalIfMinimized(true) for use in SLOT() macro */ + /** Simply calls showNormalIfMinimized(true) */ void toggleHidden(); /** called by a timer to check if ShutdownRequested() has been set **/ @@ -341,7 +342,7 @@ private: private Q_SLOTS: /** When Display Units are changed on OptionsModel it will refresh the display text of the control on the status bar */ - void updateDisplayUnit(int newUnits); + void updateDisplayUnit(BitcoinUnit newUnits); /** Tells underlying optionsModel to update its current display unit. */ void onMenuSelection(QAction* action); }; diff --git a/src/qt/bitcoinunits.cpp b/src/qt/bitcoinunits.cpp index 69caf64d5c..fe3eb3240b 100644 --- a/src/qt/bitcoinunits.cpp +++ b/src/qt/bitcoinunits.cpp @@ -18,94 +18,75 @@ BitcoinUnits::BitcoinUnits(QObject *parent): { } -QList<BitcoinUnits::Unit> BitcoinUnits::availableUnits() +QList<BitcoinUnit> BitcoinUnits::availableUnits() { - QList<BitcoinUnits::Unit> unitlist; - unitlist.append(BTC); - unitlist.append(mBTC); - unitlist.append(uBTC); - unitlist.append(SAT); + QList<BitcoinUnit> unitlist; + unitlist.append(Unit::BTC); + unitlist.append(Unit::mBTC); + unitlist.append(Unit::uBTC); + unitlist.append(Unit::SAT); return unitlist; } -bool BitcoinUnits::valid(int unit) +QString BitcoinUnits::longName(Unit unit) { - switch(unit) - { - case BTC: - case mBTC: - case uBTC: - case SAT: - return true; - default: - return false; - } -} - -QString BitcoinUnits::longName(int unit) -{ - switch(unit) - { - case BTC: return QString("BTC"); - case mBTC: return QString("mBTC"); - case uBTC: return QString::fromUtf8("µBTC (bits)"); - case SAT: return QString("Satoshi (sat)"); - default: return QString("???"); - } + switch (unit) { + case Unit::BTC: return QString("BTC"); + case Unit::mBTC: return QString("mBTC"); + case Unit::uBTC: return QString::fromUtf8("µBTC (bits)"); + case Unit::SAT: return QString("Satoshi (sat)"); + } // no default case, so the compiler can warn about missing cases + assert(false); } -QString BitcoinUnits::shortName(int unit) +QString BitcoinUnits::shortName(Unit unit) { - switch(unit) - { - case uBTC: return QString::fromUtf8("bits"); - case SAT: return QString("sat"); - default: return longName(unit); - } + switch (unit) { + case Unit::BTC: return longName(unit); + case Unit::mBTC: return longName(unit); + case Unit::uBTC: return QString("bits"); + case Unit::SAT: return QString("sat"); + } // no default case, so the compiler can warn about missing cases + assert(false); } -QString BitcoinUnits::description(int unit) +QString BitcoinUnits::description(Unit unit) { - switch(unit) - { - case BTC: return QString("Bitcoins"); - case mBTC: return QString("Milli-Bitcoins (1 / 1" THIN_SP_UTF8 "000)"); - case uBTC: return QString("Micro-Bitcoins (bits) (1 / 1" THIN_SP_UTF8 "000" THIN_SP_UTF8 "000)"); - case SAT: return QString("Satoshi (sat) (1 / 100" THIN_SP_UTF8 "000" THIN_SP_UTF8 "000)"); - default: return QString("???"); - } + switch (unit) { + case Unit::BTC: return QString("Bitcoins"); + case Unit::mBTC: return QString("Milli-Bitcoins (1 / 1" THIN_SP_UTF8 "000)"); + case Unit::uBTC: return QString("Micro-Bitcoins (bits) (1 / 1" THIN_SP_UTF8 "000" THIN_SP_UTF8 "000)"); + case Unit::SAT: return QString("Satoshi (sat) (1 / 100" THIN_SP_UTF8 "000" THIN_SP_UTF8 "000)"); + } // no default case, so the compiler can warn about missing cases + assert(false); } -qint64 BitcoinUnits::factor(int unit) +qint64 BitcoinUnits::factor(Unit unit) { - switch(unit) - { - case BTC: return 100000000; - case mBTC: return 100000; - case uBTC: return 100; - case SAT: return 1; - default: return 100000000; - } + switch (unit) { + case Unit::BTC: return 100'000'000; + case Unit::mBTC: return 100'000; + case Unit::uBTC: return 100; + case Unit::SAT: return 1; + } // no default case, so the compiler can warn about missing cases + assert(false); } -int BitcoinUnits::decimals(int unit) +int BitcoinUnits::decimals(Unit unit) { - switch(unit) - { - case BTC: return 8; - case mBTC: return 5; - case uBTC: return 2; - case SAT: return 0; - default: return 0; - } + switch (unit) { + case Unit::BTC: return 8; + case Unit::mBTC: return 5; + case Unit::uBTC: return 2; + case Unit::SAT: return 0; + } // no default case, so the compiler can warn about missing cases + assert(false); } -QString BitcoinUnits::format(int unit, const CAmount& nIn, bool fPlus, SeparatorStyle separators, bool justify) +QString BitcoinUnits::format(Unit unit, const CAmount& nIn, bool fPlus, SeparatorStyle separators, bool justify) { // Note: not using straight sprintf here because we do NOT want // localized number formatting. - if(!valid(unit)) - return QString(); // Refuse to format invalid unit qint64 n = (qint64)nIn; qint64 coin = factor(unit); int num_decimals = decimals(unit); @@ -147,19 +128,19 @@ QString BitcoinUnits::format(int unit, const CAmount& nIn, bool fPlus, Separator // Please take care to use formatHtmlWithUnit instead, when // appropriate. -QString BitcoinUnits::formatWithUnit(int unit, const CAmount& amount, bool plussign, SeparatorStyle separators) +QString BitcoinUnits::formatWithUnit(Unit unit, const CAmount& amount, bool plussign, SeparatorStyle separators) { return format(unit, amount, plussign, separators) + QString(" ") + shortName(unit); } -QString BitcoinUnits::formatHtmlWithUnit(int unit, const CAmount& amount, bool plussign, SeparatorStyle separators) +QString BitcoinUnits::formatHtmlWithUnit(Unit unit, const CAmount& amount, bool plussign, SeparatorStyle separators) { QString str(formatWithUnit(unit, amount, plussign, separators)); str.replace(QChar(THIN_SP_CP), QString(THIN_SP_HTML)); return QString("<span style='white-space: nowrap;'>%1</span>").arg(str); } -QString BitcoinUnits::formatWithPrivacy(int unit, const CAmount& amount, SeparatorStyle separators, bool privacy) +QString BitcoinUnits::formatWithPrivacy(Unit unit, const CAmount& amount, SeparatorStyle separators, bool privacy) { assert(amount >= 0); QString value; @@ -171,10 +152,11 @@ QString BitcoinUnits::formatWithPrivacy(int unit, const CAmount& amount, Separat return value + QString(" ") + shortName(unit); } -bool BitcoinUnits::parse(int unit, const QString &value, CAmount *val_out) +bool BitcoinUnits::parse(Unit unit, const QString& value, CAmount* val_out) { - if(!valid(unit) || value.isEmpty()) + if (value.isEmpty()) { return false; // Refuse to parse invalid unit or empty string + } int num_decimals = decimals(unit); // Ignore spaces and thin spaces when parsing @@ -210,14 +192,9 @@ bool BitcoinUnits::parse(int unit, const QString &value, CAmount *val_out) return ok; } -QString BitcoinUnits::getAmountColumnTitle(int unit) +QString BitcoinUnits::getAmountColumnTitle(Unit unit) { - QString amountTitle = QObject::tr("Amount"); - if (BitcoinUnits::valid(unit)) - { - amountTitle += " ("+BitcoinUnits::shortName(unit) + ")"; - } - return amountTitle; + return QObject::tr("Amount") + " (" + shortName(unit) + ")"; } int BitcoinUnits::rowCount(const QModelIndex &parent) const @@ -240,7 +217,7 @@ QVariant BitcoinUnits::data(const QModelIndex &index, int role) const case Qt::ToolTipRole: return QVariant(description(unit)); case UnitRole: - return QVariant(static_cast<int>(unit)); + return QVariant::fromValue(unit); } } return QVariant(); @@ -250,3 +227,40 @@ CAmount BitcoinUnits::maxMoney() { return MAX_MONEY; } + +namespace { +qint8 ToQint8(BitcoinUnit unit) +{ + switch (unit) { + case BitcoinUnit::BTC: return 0; + case BitcoinUnit::mBTC: return 1; + case BitcoinUnit::uBTC: return 2; + case BitcoinUnit::SAT: return 3; + } // no default case, so the compiler can warn about missing cases + assert(false); +} + +BitcoinUnit FromQint8(qint8 num) +{ + switch (num) { + case 0: return BitcoinUnit::BTC; + case 1: return BitcoinUnit::mBTC; + case 2: return BitcoinUnit::uBTC; + case 3: return BitcoinUnit::SAT; + } + assert(false); +} +} // namespace + +QDataStream& operator<<(QDataStream& out, const BitcoinUnit& unit) +{ + return out << ToQint8(unit); +} + +QDataStream& operator>>(QDataStream& in, BitcoinUnit& unit) +{ + qint8 input; + in >> input; + unit = FromQint8(input); + return in; +} diff --git a/src/qt/bitcoinunits.h b/src/qt/bitcoinunits.h index 9fedec0d4f..b3b5a8fc18 100644 --- a/src/qt/bitcoinunits.h +++ b/src/qt/bitcoinunits.h @@ -8,6 +8,7 @@ #include <consensus/amount.h> #include <QAbstractListModel> +#include <QDataStream> #include <QString> // U+2009 THIN SPACE = UTF-8 E2 80 89 @@ -38,13 +39,13 @@ public: /** Bitcoin units. @note Source: https://en.bitcoin.it/wiki/Units . Please add only sensible ones */ - enum Unit - { + enum class Unit { BTC, mBTC, uBTC, SAT }; + Q_ENUM(Unit) enum class SeparatorStyle { @@ -59,30 +60,28 @@ public: //! Get list of units, for drop-down box static QList<Unit> availableUnits(); - //! Is unit ID valid? - static bool valid(int unit); //! Long name - static QString longName(int unit); + static QString longName(Unit unit); //! Short name - static QString shortName(int unit); + static QString shortName(Unit unit); //! Longer description - static QString description(int unit); + static QString description(Unit unit); //! Number of Satoshis (1e-8) per unit - static qint64 factor(int unit); + static qint64 factor(Unit unit); //! Number of decimals left - static int decimals(int unit); + static int decimals(Unit unit); //! Format as string - static QString format(int unit, const CAmount& amount, bool plussign = false, SeparatorStyle separators = SeparatorStyle::STANDARD, bool justify = false); + static QString format(Unit unit, const CAmount& amount, bool plussign = false, SeparatorStyle separators = SeparatorStyle::STANDARD, bool justify = false); //! Format as string (with unit) - static QString formatWithUnit(int unit, const CAmount& amount, bool plussign=false, SeparatorStyle separators=SeparatorStyle::STANDARD); + static QString formatWithUnit(Unit unit, const CAmount& amount, bool plussign = false, SeparatorStyle separators = SeparatorStyle::STANDARD); //! Format as HTML string (with unit) - static QString formatHtmlWithUnit(int unit, const CAmount& amount, bool plussign=false, SeparatorStyle separators=SeparatorStyle::STANDARD); + static QString formatHtmlWithUnit(Unit unit, const CAmount& amount, bool plussign = false, SeparatorStyle separators = SeparatorStyle::STANDARD); //! Format as string (with unit) of fixed length to preserve privacy, if it is set. - static QString formatWithPrivacy(int unit, const CAmount& amount, SeparatorStyle separators, bool privacy); + static QString formatWithPrivacy(Unit unit, const CAmount& amount, SeparatorStyle separators, bool privacy); //! Parse string to coin amount - static bool parse(int unit, const QString &value, CAmount *val_out); + static bool parse(Unit unit, const QString& value, CAmount* val_out); //! Gets title for amount column including current display unit if optionsModel reference available */ - static QString getAmountColumnTitle(int unit); + static QString getAmountColumnTitle(Unit unit); ///@} //! @name AbstractListModel implementation @@ -107,8 +106,11 @@ public: static CAmount maxMoney(); private: - QList<BitcoinUnits::Unit> unitlist; + QList<Unit> unitlist; }; typedef BitcoinUnits::Unit BitcoinUnit; +QDataStream& operator<<(QDataStream& out, const BitcoinUnit& unit); +QDataStream& operator>>(QDataStream& in, BitcoinUnit& unit); + #endif // BITCOIN_QT_BITCOINUNITS_H diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index d7a2aaaf19..bd9a90a890 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -16,10 +16,11 @@ #include <qt/platformstyle.h> #include <qt/walletmodel.h> -#include <wallet/coincontrol.h> #include <interfaces/node.h> #include <key_io.h> #include <policy/policy.h> +#include <wallet/coincontrol.h> +#include <wallet/coinselection.h> #include <wallet/wallet.h> #include <QApplication> @@ -32,7 +33,6 @@ #include <QTreeWidget> using wallet::CCoinControl; -using wallet::MIN_CHANGE; QList<CAmount> CoinControlDialog::payAmounts; bool CoinControlDialog::fSubtractFeeFromAmount = false; @@ -485,11 +485,10 @@ void CoinControlDialog::updateLabels(CCoinControl& m_coin_control, WalletModel * if (!CoinControlDialog::fSubtractFeeFromAmount) nChange -= nPayFee; - // Never create dust outputs; if we would, just add the dust to the fee. - if (nChange > 0 && nChange < MIN_CHANGE) - { + if (nChange > 0) { // Assumes a p2pkh script size CTxOut txout(nChange, CScript() << std::vector<unsigned char>(24, 0)); + // Never create dust outputs; if we would, just add the dust to the fee. if (IsDust(txout, model->node().getDustRelayFee())) { nPayFee += nChange; @@ -508,7 +507,7 @@ void CoinControlDialog::updateLabels(CCoinControl& m_coin_control, WalletModel * } // actually update labels - int nDisplayUnit = BitcoinUnits::BTC; + BitcoinUnit nDisplayUnit = BitcoinUnit::BTC; if (model && model->getOptionsModel()) nDisplayUnit = model->getOptionsModel()->getDisplayUnit(); @@ -589,9 +588,9 @@ void CoinControlDialog::updateView() ui->treeWidget->setEnabled(false); // performance, otherwise updateLabels would be called for every checked checkbox ui->treeWidget->setAlternatingRowColors(!treeMode); QFlags<Qt::ItemFlag> flgCheckbox = Qt::ItemIsSelectable | Qt::ItemIsEnabled | Qt::ItemIsUserCheckable; - QFlags<Qt::ItemFlag> flgTristate = Qt::ItemIsSelectable | Qt::ItemIsEnabled | Qt::ItemIsUserCheckable | Qt::ItemIsTristate; + QFlags<Qt::ItemFlag> flgTristate = Qt::ItemIsSelectable | Qt::ItemIsEnabled | Qt::ItemIsUserCheckable | Qt::ItemIsAutoTristate; - int nDisplayUnit = model->getOptionsModel()->getDisplayUnit(); + BitcoinUnit nDisplayUnit = model->getOptionsModel()->getDisplayUnit(); for (const auto& coins : model->wallet().listCoins()) { CCoinControlWidgetItem* itemWalletAddress{nullptr}; diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui index 2196801023..ead977296a 100644 --- a/src/qt/forms/debugwindow.ui +++ b/src/qt/forms/debugwindow.ui @@ -1594,10 +1594,10 @@ <item row="23" column="0"> <widget class="QLabel" name="peerAddrRelayEnabledLabel"> <property name="toolTip"> - <string extracomment="Tooltip text for the Address Relay field in the peer details area.">Whether we relay addresses to this peer.</string> + <string extracomment="Tooltip text for the Address Relay field in the peer details area, which displays whether we relay addresses to this peer (Yes/No).">Whether we relay addresses to this peer.</string> </property> <property name="text"> - <string>Address Relay</string> + <string extracomment="Text title for the Address Relay field in the peer details area, which displays whether we relay addresses to this peer (Yes/No).">Address Relay</string> </property> </widget> </item> @@ -1620,10 +1620,10 @@ <item row="24" column="0"> <widget class="QLabel" name="peerAddrProcessedLabel"> <property name="toolTip"> - <string extracomment="Tooltip text for the Addresses Processed field in the peer details area.">Total number of addresses processed, excluding those dropped due to rate-limiting.</string> + <string extracomment="Tooltip text for the Addresses Processed field in the peer details area, which displays the total number of addresses received from this peer that were processed (excludes addresses that were dropped due to rate-limiting).">The total number of addresses received from this peer that were processed (excludes addresses that were dropped due to rate-limiting).</string> </property> <property name="text"> - <string>Addresses Processed</string> + <string extracomment="Text title for the Addresses Processed field in the peer details area, which displays the total number of addresses received from this peer that were processed (excludes addresses that were dropped due to rate-limiting).">Addresses Processed</string> </property> </widget> </item> @@ -1646,10 +1646,10 @@ <item row="25" column="0"> <widget class="QLabel" name="peerAddrRateLimitedLabel"> <property name="toolTip"> - <string extracomment="Tooltip text for the Addresses Rate-Limited field in the peer details area.">Total number of addresses dropped due to rate-limiting.</string> + <string extracomment="Tooltip text for the Addresses Rate-Limited field in the peer details area, which displays the total number of addresses received from this peer that were dropped (not processed) due to rate-limiting.">The total number of addresses received from this peer that were dropped (not processed) due to rate-limiting.</string> </property> <property name="text"> - <string>Addresses Rate-Limited</string> + <string extracomment="Text title for the Addresses Rate-Limited field in the peer details area, which displays the total number of addresses received from this peer that were dropped (not processed) due to rate-limiting.">Addresses Rate-Limited</string> </property> </widget> </item> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 9565fa508f..706cf2f7db 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -47,6 +47,7 @@ #include <QGuiApplication> #include <QJsonObject> #include <QKeyEvent> +#include <QKeySequence> #include <QLatin1String> #include <QLineEdit> #include <QList> @@ -59,6 +60,7 @@ #include <QSettings> #include <QShortcut> #include <QSize> +#include <QStandardPaths> #include <QString> #include <QTextDocument> // for Qt::mightBeRichText #include <QThread> @@ -79,6 +81,8 @@ void ForceActivation(); #endif +using namespace std::chrono_literals; + namespace GUIUtil { QString dateTimeStr(const QDateTime &date) @@ -174,8 +178,7 @@ bool parseBitcoinURI(const QUrl &uri, SendCoinsRecipient *out) { if(!i->second.isEmpty()) { - if(!BitcoinUnits::parse(BitcoinUnits::BTC, i->second, &rv.amount)) - { + if (!BitcoinUnits::parse(BitcoinUnit::BTC, i->second, &rv.amount)) { return false; } } @@ -207,7 +210,7 @@ QString formatBitcoinURI(const SendCoinsRecipient &info) if (info.amount) { - ret += QString("?amount=%1").arg(BitcoinUnits::format(BitcoinUnits::BTC, info.amount, false, BitcoinUnits::SeparatorStyle::NEVER)); + ret += QString("?amount=%1").arg(BitcoinUnits::format(BitcoinUnit::BTC, info.amount, false, BitcoinUnits::SeparatorStyle::NEVER)); paramCount++; } @@ -414,7 +417,7 @@ void bringToFront(QWidget* w) void handleCloseWindowShortcut(QWidget* w) { - QObject::connect(new QShortcut(QKeySequence(Qt::CTRL + Qt::Key_W), w), &QShortcut::activated, w, &QWidget::close); + QObject::connect(new QShortcut(QKeySequence(QObject::tr("Ctrl+W")), w), &QShortcut::activated, w, &QWidget::close); } void openDebugLogfile() @@ -505,7 +508,7 @@ fs::path static StartupShortcutPath() return GetSpecialFolderPath(CSIDL_STARTUP) / "Bitcoin.lnk"; if (chain == CBaseChainParams::TESTNET) // Remove this special case when CBaseChainParams::TESTNET = "testnet4" return GetSpecialFolderPath(CSIDL_STARTUP) / "Bitcoin (testnet).lnk"; - return GetSpecialFolderPath(CSIDL_STARTUP) / strprintf("Bitcoin (%s).lnk", chain); + return GetSpecialFolderPath(CSIDL_STARTUP) / fs::u8path(strprintf("Bitcoin (%s).lnk", chain)); } bool GetStartOnSystemStartup() @@ -586,7 +589,7 @@ fs::path static GetAutostartFilePath() std::string chain = gArgs.GetChainName(); if (chain == CBaseChainParams::MAIN) return GetAutostartDir() / "bitcoin.desktop"; - return GetAutostartDir() / strprintf("bitcoin-%s.desktop", chain); + return GetAutostartDir() / fs::u8path(strprintf("bitcoin-%s.desktop", chain)); } bool GetStartOnSystemStartup() @@ -713,23 +716,28 @@ QString ConnectionTypeToQString(ConnectionType conn_type, bool prepend_direction QString formatDurationStr(std::chrono::seconds dur) { - const auto secs = count_seconds(dur); - QStringList strList; - int days = secs / 86400; - int hours = (secs % 86400) / 3600; - int mins = (secs % 3600) / 60; - int seconds = secs % 60; - - if (days) - strList.append(QObject::tr("%1 d").arg(days)); - if (hours) - strList.append(QObject::tr("%1 h").arg(hours)); - if (mins) - strList.append(QObject::tr("%1 m").arg(mins)); - if (seconds || (!days && !hours && !mins)) - strList.append(QObject::tr("%1 s").arg(seconds)); + using days = std::chrono::duration<int, std::ratio<86400>>; // can remove this line after C++20 + const auto d{std::chrono::duration_cast<days>(dur)}; + const auto h{std::chrono::duration_cast<std::chrono::hours>(dur - d)}; + const auto m{std::chrono::duration_cast<std::chrono::minutes>(dur - d - h)}; + const auto s{std::chrono::duration_cast<std::chrono::seconds>(dur - d - h - m)}; + QStringList str_list; + if (auto d2{d.count()}) str_list.append(QObject::tr("%1 d").arg(d2)); + if (auto h2{h.count()}) str_list.append(QObject::tr("%1 h").arg(h2)); + if (auto m2{m.count()}) str_list.append(QObject::tr("%1 m").arg(m2)); + const auto s2{s.count()}; + if (s2 || str_list.empty()) str_list.append(QObject::tr("%1 s").arg(s2)); + return str_list.join(" "); +} - return strList.join(" "); +QString FormatPeerAge(std::chrono::seconds time_connected) +{ + const auto time_now{GetTime<std::chrono::seconds>()}; + const auto age{time_now - time_connected}; + if (age >= 24h) return QObject::tr("%1 d").arg(age / 24h); + if (age >= 1h) return QObject::tr("%1 h").arg(age / 1h); + if (age >= 1min) return QObject::tr("%1 m").arg(age / 1min); + return QObject::tr("%1 s").arg(age / 1s); } QString formatServicesStr(quint64 mask) @@ -888,11 +896,7 @@ void PolishProgressDialog(QProgressDialog* dialog) int TextWidth(const QFontMetrics& fm, const QString& text) { -#if (QT_VERSION >= QT_VERSION_CHECK(5, 11, 0)) return fm.horizontalAdvance(text); -#else - return fm.width(text); -#endif } void LogQtInfo() diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index 0224b18b4e..e38ac6026a 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -223,6 +223,9 @@ namespace GUIUtil /** Convert seconds into a QString with days, hours, mins, secs */ QString formatDurationStr(std::chrono::seconds dur); + /** Convert peer connection time to a QString denominated in the most relevant unit. */ + QString FormatPeerAge(std::chrono::seconds time_connected); + /** Format CNodeStats.nServices bitmask into a user-readable string */ QString formatServicesStr(quint64 mask); @@ -358,18 +361,6 @@ namespace GUIUtil #endif } - /** - * Queue a function to run in an object's event loop. This can be - * replaced by a call to the QMetaObject::invokeMethod functor overload after Qt 5.10, but - * for now use a QObject::connect for compatibility with older Qt versions, based on - * https://stackoverflow.com/questions/21646467/how-to-execute-a-functor-or-a-lambda-in-a-given-thread-in-qt-gcd-style - */ - template <typename Fn> - void ObjectInvoke(QObject* object, Fn&& function, Qt::ConnectionType connection = Qt::QueuedConnection) - { - QObject source; - QObject::connect(&source, &QObject::destroyed, object, std::forward<Fn>(function), connection); - } /** * Replaces a plain text link with an HTML tagged one. diff --git a/src/qt/initexecutor.cpp b/src/qt/initexecutor.cpp index 24ae7ba73d..d269dfec71 100644 --- a/src/qt/initexecutor.cpp +++ b/src/qt/initexecutor.cpp @@ -5,13 +5,13 @@ #include <qt/initexecutor.h> #include <interfaces/node.h> -#include <qt/guiutil.h> #include <util/system.h> #include <util/threadnames.h> #include <exception> #include <QDebug> +#include <QMetaObject> #include <QObject> #include <QString> #include <QThread> @@ -39,7 +39,7 @@ void InitExecutor::handleRunawayException(const std::exception* e) void InitExecutor::initialize() { - GUIUtil::ObjectInvoke(&m_context, [this] { + QMetaObject::invokeMethod(&m_context, [this] { try { util::ThreadRename("qt-init"); qDebug() << "Running initialization in thread"; @@ -56,7 +56,7 @@ void InitExecutor::initialize() void InitExecutor::shutdown() { - GUIUtil::ObjectInvoke(&m_context, [this] { + QMetaObject::invokeMethod(&m_context, [this] { try { qDebug() << "Running Shutdown in thread"; m_node.appShutdown(); diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index dcde7adec4..f4928951fe 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -226,7 +226,7 @@ bool Intro::showIfNeeded(bool& did_show_intro, int64_t& prune_MiB) } /* If current default data directory does not exist, let the user choose one */ - Intro intro(0, Params().AssumedBlockchainSize(), Params().AssumedChainStateSize()); + Intro intro(nullptr, Params().AssumedBlockchainSize(), Params().AssumedChainStateSize()); intro.setDataDirectory(dataDir); intro.setWindowIcon(QIcon(":icons/bitcoin")); did_show_intro = true; @@ -298,12 +298,12 @@ void Intro::setStatus(int status, const QString &message, quint64 bytesAvailable void Intro::UpdateFreeSpaceLabel() { - QString freeString = tr("%1 GB of space available").arg(m_bytes_available / GB_BYTES); + QString freeString = tr("%n GB of space available", "", m_bytes_available / GB_BYTES); if (m_bytes_available < m_required_space_gb * GB_BYTES) { - freeString += " " + tr("(of %1 GB needed)").arg(m_required_space_gb); + freeString += " " + tr("(of %n GB needed)", "", m_required_space_gb); ui->freeSpace->setStyleSheet("QLabel { color: #800000 }"); } else if (m_bytes_available / GB_BYTES - m_required_space_gb < 10) { - freeString += " " + tr("(%1 GB needed for full chain)").arg(m_required_space_gb); + freeString += " " + tr("(%n GB needed for full chain)", "", m_required_space_gb); ui->freeSpace->setStyleSheet("QLabel { color: #999900 }"); } else { ui->freeSpace->setStyleSheet(""); diff --git a/src/qt/notificator.cpp b/src/qt/notificator.cpp index b097ef080c..51151b0be8 100644 --- a/src/qt/notificator.cpp +++ b/src/qt/notificator.cpp @@ -14,8 +14,9 @@ #include <QTemporaryFile> #include <QVariant> #ifdef USE_DBUS -#include <stdint.h> +#include <QDBusMetaType> #include <QtDBus> +#include <stdint.h> #endif #ifdef Q_OS_MAC #include <qt/macnotificationhandler.h> @@ -73,8 +74,6 @@ public: FreedesktopImage() {} explicit FreedesktopImage(const QImage &img); - static int metaType(); - // Image to variant that can be marshalled over DBus static QVariant toVariant(const QImage &img); @@ -136,15 +135,10 @@ const QDBusArgument &operator>>(const QDBusArgument &a, FreedesktopImage &i) return a; } -int FreedesktopImage::metaType() -{ - return qDBusRegisterMetaType<FreedesktopImage>(); -} - QVariant FreedesktopImage::toVariant(const QImage &img) { FreedesktopImage fimg(img); - return QVariant(FreedesktopImage::metaType(), &fimg); + return QVariant(qDBusRegisterMetaType<FreedesktopImage>(), &fimg); } void Notificator::notifyDBus(Class cls, const QString &title, const QString &text, const QIcon &icon, int millisTimeout) diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index 5d9ed5bf23..40b9ed5483 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -24,6 +24,7 @@ #include <QLatin1Char> #include <QSettings> #include <QStringList> +#include <QVariant> const char *DEFAULT_GUI_PROXY_HOST = "127.0.0.1"; @@ -71,9 +72,16 @@ void OptionsModel::Init(bool resetSettings) fMinimizeOnClose = settings.value("fMinimizeOnClose").toBool(); // Display - if (!settings.contains("nDisplayUnit")) - settings.setValue("nDisplayUnit", BitcoinUnits::BTC); - nDisplayUnit = settings.value("nDisplayUnit").toInt(); + if (!settings.contains("DisplayBitcoinUnit")) { + settings.setValue("DisplayBitcoinUnit", QVariant::fromValue(BitcoinUnit::BTC)); + } + QVariant unit = settings.value("DisplayBitcoinUnit"); + if (unit.canConvert<BitcoinUnit>()) { + m_display_bitcoin_unit = unit.value<BitcoinUnit>(); + } else { + m_display_bitcoin_unit = BitcoinUnit::BTC; + settings.setValue("DisplayBitcoinUnit", QVariant::fromValue(m_display_bitcoin_unit)); + } if (!settings.contains("strThirdPartyTxUrls")) settings.setValue("strThirdPartyTxUrls", ""); @@ -151,8 +159,28 @@ void OptionsModel::Init(bool resetSettings) if (!settings.contains("fListen")) settings.setValue("fListen", DEFAULT_LISTEN); - if (!gArgs.SoftSetBoolArg("-listen", settings.value("fListen").toBool())) + const bool listen{settings.value("fListen").toBool()}; + if (!gArgs.SoftSetBoolArg("-listen", listen)) { addOverriddenOption("-listen"); + } else if (!listen) { + // We successfully set -listen=0, thus mimic the logic from InitParameterInteraction(): + // "parameter interaction: -listen=0 -> setting -listenonion=0". + // + // Both -listen and -listenonion default to true. + // + // The call order is: + // + // InitParameterInteraction() + // would set -listenonion=0 if it sees -listen=0, but for bitcoin-qt with + // fListen=false -listen is 1 at this point + // + // OptionsModel::Init() + // (this method) can flip -listen from 1 to 0 if fListen=false + // + // AppInitParameterInteraction() + // raises an error if -listen=0 and -listenonion=1 + gArgs.SoftSetBoolArg("-listenonion", false); + } if (!settings.contains("server")) { settings.setValue("server", false); @@ -356,7 +384,7 @@ QVariant OptionsModel::data(const QModelIndex & index, int role) const return m_sub_fee_from_amount; #endif case DisplayUnit: - return nDisplayUnit; + return QVariant::fromValue(m_display_bitcoin_unit); case ThirdPartyTxUrls: return strThirdPartyTxUrls; case Language: @@ -564,16 +592,13 @@ bool OptionsModel::setData(const QModelIndex & index, const QVariant & value, in return successful; } -/** Updates current unit in memory, settings and emits displayUnitChanged(newUnit) signal */ -void OptionsModel::setDisplayUnit(const QVariant &value) +void OptionsModel::setDisplayUnit(const QVariant& new_unit) { - if (!value.isNull()) - { - QSettings settings; - nDisplayUnit = value.toInt(); - settings.setValue("nDisplayUnit", nDisplayUnit); - Q_EMIT displayUnitChanged(nDisplayUnit); - } + if (new_unit.isNull() || new_unit.value<BitcoinUnit>() == m_display_bitcoin_unit) return; + m_display_bitcoin_unit = new_unit.value<BitcoinUnit>(); + QSettings settings; + settings.setValue("DisplayBitcoinUnit", QVariant::fromValue(m_display_bitcoin_unit)); + Q_EMIT displayUnitChanged(m_display_bitcoin_unit); } void OptionsModel::setRestartRequired(bool fRequired) diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index bb9a8c1f8c..510ebb5cfd 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -6,6 +6,7 @@ #define BITCOIN_QT_OPTIONSMODEL_H #include <cstdint> +#include <qt/bitcoinunits.h> #include <qt/guiconstants.h> #include <QAbstractListModel> @@ -55,7 +56,7 @@ public: ProxyUseTor, // bool ProxyIPTor, // QString ProxyPortTor, // int - DisplayUnit, // BitcoinUnits::Unit + DisplayUnit, // BitcoinUnit ThirdPartyTxUrls, // QString Language, // QString UseEmbeddedMonospacedFont, // bool @@ -79,14 +80,14 @@ public: int rowCount(const QModelIndex & parent = QModelIndex()) const override; QVariant data(const QModelIndex & index, int role = Qt::DisplayRole) const override; bool setData(const QModelIndex & index, const QVariant & value, int role = Qt::EditRole) override; - /** Updates current unit in memory, settings and emits displayUnitChanged(newUnit) signal */ - void setDisplayUnit(const QVariant &value); + /** Updates current unit in memory, settings and emits displayUnitChanged(new_unit) signal */ + void setDisplayUnit(const QVariant& new_unit); /* Explicit getters */ bool getShowTrayIcon() const { return m_show_tray_icon; } bool getMinimizeToTray() const { return fMinimizeToTray; } bool getMinimizeOnClose() const { return fMinimizeOnClose; } - int getDisplayUnit() const { return nDisplayUnit; } + BitcoinUnit getDisplayUnit() const { return m_display_bitcoin_unit; } QString getThirdPartyTxUrls() const { return strThirdPartyTxUrls; } bool getUseEmbeddedMonospacedFont() const { return m_use_embedded_monospaced_font; } bool getCoinControlFeatures() const { return fCoinControlFeatures; } @@ -112,7 +113,7 @@ private: bool fMinimizeToTray; bool fMinimizeOnClose; QString language; - int nDisplayUnit; + BitcoinUnit m_display_bitcoin_unit; QString strThirdPartyTxUrls; bool m_use_embedded_monospaced_font; bool fCoinControlFeatures; @@ -127,7 +128,7 @@ private: // Check settings version and upgrade default values if required void checkAndMigrate(); Q_SIGNALS: - void displayUnitChanged(int unit); + void displayUnitChanged(BitcoinUnit unit); void coinControlFeaturesChanged(bool); void showTrayIconChanged(bool); void useEmbeddedMonospacedFontChanged(bool); diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index 7127706463..820bcbf3cd 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -34,9 +34,9 @@ class TxViewDelegate : public QAbstractItemDelegate { Q_OBJECT public: - explicit TxViewDelegate(const PlatformStyle *_platformStyle, QObject *parent=nullptr): - QAbstractItemDelegate(parent), unit(BitcoinUnits::BTC), - platformStyle(_platformStyle) + explicit TxViewDelegate(const PlatformStyle* _platformStyle, QObject* parent = nullptr) + : QAbstractItemDelegate(parent), unit(BitcoinUnit::BTC), + platformStyle(_platformStyle) { connect(this, &TxViewDelegate::width_changed, this, &TxViewDelegate::sizeHintChanged); } @@ -125,7 +125,7 @@ public: return {DECORATION_SIZE + 8 + minimum_text_width, DECORATION_SIZE}; } - int unit; + BitcoinUnit unit; Q_SIGNALS: //! An intermediate signal for emitting from the `paint() const` member function. @@ -197,7 +197,7 @@ OverviewPage::~OverviewPage() void OverviewPage::setBalance(const interfaces::WalletBalances& balances) { - int unit = walletModel->getOptionsModel()->getDisplayUnit(); + BitcoinUnit unit = walletModel->getOptionsModel()->getDisplayUnit(); m_balances = balances; if (walletModel->wallet().isLegacy()) { if (walletModel->wallet().privateKeysDisabled()) { diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index cb09035b45..c82f0683fc 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -78,32 +78,11 @@ void PaymentServer::ipcParseCommandLine(int argc, char* argv[]) for (int i = 1; i < argc; i++) { QString arg(argv[i]); - if (arg.startsWith("-")) - continue; + if (arg.startsWith("-")) continue; - // If the bitcoin: URI contains a payment request, we are not able to detect the - // network as that would require fetching and parsing the payment request. - // That means clicking such an URI which contains a testnet payment request - // will start a mainnet instance and throw a "wrong network" error. if (arg.startsWith(BITCOIN_IPC_PREFIX, Qt::CaseInsensitive)) // bitcoin: URI { - if (savedPaymentRequests.contains(arg)) continue; savedPaymentRequests.insert(arg); - - SendCoinsRecipient r; - if (GUIUtil::parseBitcoinURI(arg, &r) && !r.address.isEmpty()) - { - auto tempChainParams = CreateChainParams(gArgs, CBaseChainParams::MAIN); - - if (IsValidDestinationString(r.address.toStdString(), *tempChainParams)) { - SelectParams(CBaseChainParams::MAIN); - } else { - tempChainParams = CreateChainParams(gArgs, CBaseChainParams::TESTNET); - if (IsValidDestinationString(r.address.toStdString(), *tempChainParams)) { - SelectParams(CBaseChainParams::TESTNET); - } - } - } } } } diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index cd3193a1d2..41c389d9cc 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -71,6 +71,8 @@ QVariant PeerTableModel::data(const QModelIndex& index, int role) const switch (column) { case NetNodeId: return (qint64)rec->nodeStats.nodeid; + case Age: + return GUIUtil::FormatPeerAge(rec->nodeStats.m_connected); case Address: return QString::fromStdString(rec->nodeStats.m_addr_name); case Direction: @@ -80,7 +82,7 @@ QVariant PeerTableModel::data(const QModelIndex& index, int role) const //: An Outbound Connection to a Peer. tr("Outbound")); case ConnectionType: - return GUIUtil::ConnectionTypeToQString(rec->nodeStats.m_conn_type, /* prepend_direction */ false); + return GUIUtil::ConnectionTypeToQString(rec->nodeStats.m_conn_type, /*prepend_direction=*/false); case Network: return GUIUtil::NetworkToQString(rec->nodeStats.m_network); case Ping: @@ -96,6 +98,7 @@ QVariant PeerTableModel::data(const QModelIndex& index, int role) const } else if (role == Qt::TextAlignmentRole) { switch (column) { case NetNodeId: + case Age: return QVariant(Qt::AlignRight | Qt::AlignVCenter); case Address: return {}; diff --git a/src/qt/peertablemodel.h b/src/qt/peertablemodel.h index 11064cdbfe..e2515de775 100644 --- a/src/qt/peertablemodel.h +++ b/src/qt/peertablemodel.h @@ -47,6 +47,7 @@ public: enum ColumnIndex { NetNodeId = 0, + Age, Address, Direction, ConnectionType, @@ -82,6 +83,9 @@ private: /*: Title of Peers Table column which contains a unique number used to identify a connection. */ tr("Peer"), + /*: Title of Peers Table column which indicates the duration (length of time) + since the peer connection started. */ + tr("Age"), /*: Title of Peers Table column which contains the IP/Onion/I2P address of the connected peer. */ tr("Address"), diff --git a/src/qt/peertablesortproxy.cpp b/src/qt/peertablesortproxy.cpp index 26fedb4127..d87f10c365 100644 --- a/src/qt/peertablesortproxy.cpp +++ b/src/qt/peertablesortproxy.cpp @@ -24,6 +24,8 @@ bool PeerTableSortProxy::lessThan(const QModelIndex& left_index, const QModelInd switch (static_cast<PeerTableModel::ColumnIndex>(left_index.column())) { case PeerTableModel::NetNodeId: return left_stats.nodeid < right_stats.nodeid; + case PeerTableModel::Age: + return left_stats.m_connected > right_stats.m_connected; case PeerTableModel::Address: return left_stats.m_addr_name.compare(right_stats.m_addr_name) < 0; case PeerTableModel::Direction: diff --git a/src/qt/psbtoperationsdialog.cpp b/src/qt/psbtoperationsdialog.cpp index 6880c157c0..333766ce21 100644 --- a/src/qt/psbtoperationsdialog.cpp +++ b/src/qt/psbtoperationsdialog.cpp @@ -181,7 +181,7 @@ std::string PSBTOperationsDialog::renderTransaction(const PartiallySignedTransac ExtractDestination(out.scriptPubKey, address); totalAmount += out.nValue; tx_description.append(tr(" * Sends %1 to %2") - .arg(BitcoinUnits::formatWithUnit(BitcoinUnits::BTC, out.nValue)) + .arg(BitcoinUnits::formatWithUnit(BitcoinUnit::BTC, out.nValue)) .arg(QString::fromStdString(EncodeDestination(address)))); tx_description.append("<br>"); } @@ -193,7 +193,7 @@ std::string PSBTOperationsDialog::renderTransaction(const PartiallySignedTransac tx_description.append(tr("Unable to calculate transaction fee or total transaction amount.")); } else { tx_description.append(tr("Pays transaction fee: ")); - tx_description.append(BitcoinUnits::formatWithUnit(BitcoinUnits::BTC, *analysis.fee)); + tx_description.append(BitcoinUnits::formatWithUnit(BitcoinUnit::BTC, *analysis.fee)); // add total amount in all subdivision units tx_description.append("<hr />"); diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index c5e5e69df6..4a51990f88 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -40,6 +40,7 @@ #include <QDateTime> #include <QFont> #include <QKeyEvent> +#include <QKeySequence> #include <QLatin1String> #include <QLocale> #include <QMenu> @@ -617,17 +618,16 @@ bool RPCConsole::eventFilter(QObject* obj, QEvent *event) case Qt::Key_Down: if(obj == ui->lineEdit) { browseHistory(1); return true; } break; case Qt::Key_PageUp: /* pass paging keys to messages widget */ case Qt::Key_PageDown: - if(obj == ui->lineEdit) - { - QApplication::postEvent(ui->messagesWidget, new QKeyEvent(*keyevt)); + if (obj == ui->lineEdit) { + QApplication::sendEvent(ui->messagesWidget, keyevt); return true; } break; case Qt::Key_Return: case Qt::Key_Enter: // forward these events to lineEdit - if(obj == autoCompleter->popup()) { - QApplication::postEvent(ui->lineEdit, new QKeyEvent(*keyevt)); + if (obj == autoCompleter->popup()) { + QApplication::sendEvent(ui->lineEdit, keyevt); autoCompleter->popup()->hide(); return true; } @@ -641,7 +641,7 @@ bool RPCConsole::eventFilter(QObject* obj, QEvent *event) ((mod & Qt::ShiftModifier) && key == Qt::Key_Insert))) { ui->lineEdit->setFocus(); - QApplication::postEvent(ui->lineEdit, new QKeyEvent(*keyevt)); + QApplication::sendEvent(ui->lineEdit, keyevt); return true; } } @@ -693,6 +693,7 @@ void RPCConsole::setClientModel(ClientModel *model, int bestblock_height, int64_ ui->peerWidget->setColumnWidth(PeerTableModel::Subversion, SUBVERSION_COLUMN_WIDTH); ui->peerWidget->setColumnWidth(PeerTableModel::Ping, PING_COLUMN_WIDTH); } + ui->peerWidget->horizontalHeader()->setSectionResizeMode(PeerTableModel::Age, QHeaderView::ResizeToContents); ui->peerWidget->horizontalHeader()->setStretchLastSection(true); ui->peerWidget->setItemDelegateForColumn(PeerTableModel::NetNodeId, new PeerIdViewDelegate(this)); @@ -725,6 +726,7 @@ void RPCConsole::setClientModel(ClientModel *model, int bestblock_height, int64_ ui->banlistWidget->setColumnWidth(BanTableModel::Address, BANSUBNET_COLUMN_WIDTH); ui->banlistWidget->setColumnWidth(BanTableModel::Bantime, BANTIME_COLUMN_WIDTH); } + ui->banlistWidget->horizontalHeader()->setSectionResizeMode(BanTableModel::Address, QHeaderView::ResizeToContents); ui->banlistWidget->horizontalHeader()->setStretchLastSection(true); // create ban table context menu @@ -847,7 +849,7 @@ void RPCConsole::setFontSize(int newSize) // clear console (reset icon sizes, default stylesheet) and re-add the content float oldPosFactor = 1.0 / ui->messagesWidget->verticalScrollBar()->maximum() * ui->messagesWidget->verticalScrollBar()->value(); - clear(/* keep_prompt */ true); + clear(/*keep_prompt=*/true); ui->messagesWidget->setHtml(str); ui->messagesWidget->verticalScrollBar()->setValue(oldPosFactor * ui->messagesWidget->verticalScrollBar()->maximum()); } @@ -1030,8 +1032,9 @@ void RPCConsole::on_lineEdit_returnPressed() ui->lineEdit->clear(); + WalletModel* wallet_model{nullptr}; #ifdef ENABLE_WALLET - WalletModel* wallet_model = ui->WalletSelector->currentData().value<WalletModel*>(); + wallet_model = ui->WalletSelector->currentData().value<WalletModel*>(); if (m_last_wallet_model != wallet_model) { if (wallet_model) { @@ -1047,7 +1050,10 @@ void RPCConsole::on_lineEdit_returnPressed() //: A console message indicating an entered command is currently being executed. message(CMD_REPLY, tr("Executing…")); m_is_executing = true; - Q_EMIT cmdRequest(cmd, m_last_wallet_model); + + QMetaObject::invokeMethod(m_executor, [this, cmd, wallet_model] { + m_executor->request(cmd, wallet_model); + }); cmd = QString::fromStdString(strFilteredCmd); @@ -1089,11 +1095,11 @@ void RPCConsole::browseHistory(int offset) void RPCConsole::startExecutor() { - RPCExecutor *executor = new RPCExecutor(m_node); - executor->moveToThread(&thread); + m_executor = new RPCExecutor(m_node); + m_executor->moveToThread(&thread); // Replies from executor object must go to this object - connect(executor, &RPCExecutor::reply, this, [this](int category, const QString& command) { + connect(m_executor, &RPCExecutor::reply, this, [this](int category, const QString& command) { // Remove "Executing…" message. ui->messagesWidget->undo(); message(category, command); @@ -1101,16 +1107,13 @@ void RPCConsole::startExecutor() m_is_executing = false; }); - // Requests from this object must go to executor - connect(this, &RPCConsole::cmdRequest, executor, &RPCExecutor::request); - // Make sure executor object is deleted in its own thread - connect(&thread, &QThread::finished, executor, &RPCExecutor::deleteLater); + connect(&thread, &QThread::finished, m_executor, &RPCExecutor::deleteLater); // Default implementation of QThread::run() simply spins up an event loop in the thread, // which is what we want. thread.start(); - QTimer::singleShot(0, executor, []() { + QTimer::singleShot(0, m_executor, []() { util::ThreadRename("qt-rpcconsole"); }); } @@ -1168,7 +1171,6 @@ void RPCConsole::updateDetailWidget() peerAddrDetails += "<br />" + tr("via %1").arg(QString::fromStdString(stats->nodeStats.addrLocal)); ui->peerHeading->setText(peerAddrDetails); ui->peerServices->setText(GUIUtil::formatServicesStr(stats->nodeStats.nServices)); - ui->peerRelayTxes->setText(stats->nodeStats.fRelayTxes ? ts.yes : ts.no); QString bip152_hb_settings; if (stats->nodeStats.m_bip152_highbandwidth_to) bip152_hb_settings = ts.to; if (stats->nodeStats.m_bip152_highbandwidth_from) bip152_hb_settings += (bip152_hb_settings.isEmpty() ? ts.from : QLatin1Char('/') + ts.from); @@ -1187,7 +1189,7 @@ void RPCConsole::updateDetailWidget() ui->timeoffset->setText(GUIUtil::formatTimeOffset(stats->nodeStats.nTimeOffset)); ui->peerVersion->setText(QString::number(stats->nodeStats.nVersion)); ui->peerSubversion->setText(QString::fromStdString(stats->nodeStats.cleanSubVer)); - ui->peerConnectionType->setText(GUIUtil::ConnectionTypeToQString(stats->nodeStats.m_conn_type, /* prepend_direction */ true)); + ui->peerConnectionType->setText(GUIUtil::ConnectionTypeToQString(stats->nodeStats.m_conn_type, /*prepend_direction=*/true)); ui->peerNetwork->setText(GUIUtil::NetworkToQString(stats->nodeStats.m_network)); if (stats->nodeStats.m_permissionFlags == NetPermissionFlags::None) { ui->peerPermissions->setText(ts.na); @@ -1220,6 +1222,7 @@ void RPCConsole::updateDetailWidget() ui->peerAddrRelayEnabled->setText(stats->nodeStateStats.m_addr_relay_enabled ? ts.yes : ts.no); ui->peerAddrProcessed->setText(QString::number(stats->nodeStateStats.m_addr_processed)); ui->peerAddrRateLimited->setText(QString::number(stats->nodeStateStats.m_addr_rate_limited)); + ui->peerRelayTxes->setText(stats->nodeStateStats.m_relay_txs ? ts.yes : ts.no); } ui->peersTabRightPanel->show(); @@ -1353,10 +1356,10 @@ QString RPCConsole::tabTitle(TabTypes tab_type) const QKeySequence RPCConsole::tabShortcut(TabTypes tab_type) const { switch (tab_type) { - case TabTypes::INFO: return QKeySequence(Qt::CTRL + Qt::Key_I); - case TabTypes::CONSOLE: return QKeySequence(Qt::CTRL + Qt::Key_T); - case TabTypes::GRAPH: return QKeySequence(Qt::CTRL + Qt::Key_N); - case TabTypes::PEERS: return QKeySequence(Qt::CTRL + Qt::Key_P); + case TabTypes::INFO: return QKeySequence(tr("Ctrl+I")); + case TabTypes::CONSOLE: return QKeySequence(tr("Ctrl+T")); + case TabTypes::GRAPH: return QKeySequence(tr("Ctrl+N")); + case TabTypes::PEERS: return QKeySequence(tr("Ctrl+P")); } // no default case, so the compiler can warn about missing cases assert(false); diff --git a/src/qt/rpcconsole.h b/src/qt/rpcconsole.h index 528e2bef7d..1a54fe0cad 100644 --- a/src/qt/rpcconsole.h +++ b/src/qt/rpcconsole.h @@ -5,6 +5,10 @@ #ifndef BITCOIN_QT_RPCCONSOLE_H #define BITCOIN_QT_RPCCONSOLE_H +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + #include <qt/guiutil.h> #include <qt/peertablemodel.h> @@ -17,6 +21,7 @@ class ClientModel; class PlatformStyle; +class RPCExecutor; class RPCTimerInterface; class WalletModel; @@ -49,8 +54,11 @@ public: } void setClientModel(ClientModel *model = nullptr, int bestblock_height = 0, int64_t bestblock_date = 0, double verification_progress = 0.0); - void addWallet(WalletModel * const walletModel); + +#ifdef ENABLE_WALLET + void addWallet(WalletModel* const walletModel); void removeWallet(WalletModel* const walletModel); +#endif // ENABLE_WALLET enum MessageClass { MC_ERROR, @@ -129,10 +137,6 @@ public Q_SLOTS: /** set which tab has the focus (is visible) */ void setTabFocus(enum TabTypes tabType); -Q_SIGNALS: - // For RPC command executor - void cmdRequest(const QString &command, const WalletModel* wallet_model); - private: struct TranslatedStrings { const QString yes{tr("Yes")}, no{tr("No")}, to{tr("To")}, from{tr("From")}, @@ -166,6 +170,7 @@ private: int consoleFontSize = 0; QCompleter *autoCompleter = nullptr; QThread thread; + RPCExecutor* m_executor{nullptr}; WalletModel* m_last_wallet_model{nullptr}; bool m_is_executing{false}; QByteArray m_peer_widget_header_state; diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 579ef0c3fd..fd8eccb86d 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -380,8 +380,7 @@ bool SendCoinsDialog::PrepareSendText(QString& question_string, QString& informa question_string.append("<hr />"); CAmount totalAmount = m_current_transaction->getTotalTransactionAmount() + txFee; QStringList alternativeUnits; - for (const BitcoinUnits::Unit u : BitcoinUnits::availableUnits()) - { + for (const BitcoinUnit u : BitcoinUnits::availableUnits()) { if(u != model->getOptionsModel()->getDisplayUnit()) alternativeUnits.append(BitcoinUnits::formatHtmlWithUnit(u, totalAmount)); } @@ -401,6 +400,80 @@ bool SendCoinsDialog::PrepareSendText(QString& question_string, QString& informa return true; } +void SendCoinsDialog::presentPSBT(PartiallySignedTransaction& psbtx) +{ + // Serialize the PSBT + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << psbtx; + GUIUtil::setClipboard(EncodeBase64(ssTx.str()).c_str()); + QMessageBox msgBox; + msgBox.setText("Unsigned Transaction"); + msgBox.setInformativeText("The PSBT has been copied to the clipboard. You can also save it."); + msgBox.setStandardButtons(QMessageBox::Save | QMessageBox::Discard); + msgBox.setDefaultButton(QMessageBox::Discard); + switch (msgBox.exec()) { + case QMessageBox::Save: { + QString selectedFilter; + QString fileNameSuggestion = ""; + bool first = true; + for (const SendCoinsRecipient &rcp : m_current_transaction->getRecipients()) { + if (!first) { + fileNameSuggestion.append(" - "); + } + QString labelOrAddress = rcp.label.isEmpty() ? rcp.address : rcp.label; + QString amount = BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), rcp.amount); + fileNameSuggestion.append(labelOrAddress + "-" + amount); + first = false; + } + fileNameSuggestion.append(".psbt"); + QString filename = GUIUtil::getSaveFileName(this, + tr("Save Transaction Data"), fileNameSuggestion, + //: Expanded name of the binary PSBT file format. See: BIP 174. + tr("Partially Signed Transaction (Binary)") + QLatin1String(" (*.psbt)"), &selectedFilter); + if (filename.isEmpty()) { + return; + } + std::ofstream out{filename.toLocal8Bit().data(), std::ofstream::out | std::ofstream::binary}; + out << ssTx.str(); + out.close(); + Q_EMIT message(tr("PSBT saved"), "PSBT saved to disk", CClientUIInterface::MSG_INFORMATION); + break; + } + case QMessageBox::Discard: + break; + default: + assert(false); + } // msgBox.exec() +} + +bool SendCoinsDialog::signWithExternalSigner(PartiallySignedTransaction& psbtx, CMutableTransaction& mtx, bool& complete) { + TransactionError err; + try { + err = model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/true, /*bip32derivs=*/true, /*n_signed=*/nullptr, psbtx, complete); + } catch (const std::runtime_error& e) { + QMessageBox::critical(nullptr, tr("Sign failed"), e.what()); + return false; + } + if (err == TransactionError::EXTERNAL_SIGNER_NOT_FOUND) { + //: "External signer" means using devices such as hardware wallets. + QMessageBox::critical(nullptr, tr("External signer not found"), "External signer not found"); + return false; + } + if (err == TransactionError::EXTERNAL_SIGNER_FAILED) { + //: "External signer" means using devices such as hardware wallets. + QMessageBox::critical(nullptr, tr("External signer failure"), "External signer failure"); + return false; + } + if (err != TransactionError::OK) { + tfm::format(std::cerr, "Failed to sign PSBT"); + processSendCoinsReturn(WalletModel::TransactionCreationFailed); + return false; + } + // fillPSBT does not always properly finalize + complete = FinalizeAndExtractPSBT(psbtx, mtx); + return true; +} + void SendCoinsDialog::sendButtonClicked([[maybe_unused]] bool checked) { if(!model || !model->getOptionsModel()) @@ -411,7 +484,9 @@ void SendCoinsDialog::sendButtonClicked([[maybe_unused]] bool checked) assert(m_current_transaction); const QString confirmation = tr("Confirm send coins"); - auto confirmationDialog = new SendConfirmationDialog(confirmation, question_string, informative_text, detailed_text, SEND_CONFIRM_DELAY, !model->wallet().privateKeysDisabled(), model->getOptionsModel()->getEnablePSBTControls(), this); + const bool enable_send{!model->wallet().privateKeysDisabled() || model->wallet().hasExternalSigner()}; + const bool always_show_unsigned{model->getOptionsModel()->getEnablePSBTControls()}; + auto confirmationDialog = new SendConfirmationDialog(confirmation, question_string, informative_text, detailed_text, SEND_CONFIRM_DELAY, enable_send, always_show_unsigned, this); confirmationDialog->setAttribute(Qt::WA_DeleteOnClose); // TODO: Replace QDialog::exec() with safer QDialog::show(). const auto retval = static_cast<QMessageBox::StandardButton>(confirmationDialog->exec()); @@ -424,49 +499,50 @@ void SendCoinsDialog::sendButtonClicked([[maybe_unused]] bool checked) bool send_failure = false; if (retval == QMessageBox::Save) { + // "Create Unsigned" clicked CMutableTransaction mtx = CMutableTransaction{*(m_current_transaction->getWtx())}; PartiallySignedTransaction psbtx(mtx); bool complete = false; - // Always fill without signing first. This prevents an external signer - // from being called prematurely and is not expensive. - TransactionError err = model->wallet().fillPSBT(SIGHASH_ALL, false /* sign */, true /* bip32derivs */, nullptr, psbtx, complete); + // Fill without signing + TransactionError err = model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, /*n_signed=*/nullptr, psbtx, complete); assert(!complete); assert(err == TransactionError::OK); + + // Copy PSBT to clipboard and offer to save + presentPSBT(psbtx); + } else { + // "Send" clicked + assert(!model->wallet().privateKeysDisabled() || model->wallet().hasExternalSigner()); + bool broadcast = true; if (model->wallet().hasExternalSigner()) { - try { - err = model->wallet().fillPSBT(SIGHASH_ALL, true /* sign */, true /* bip32derivs */, nullptr, psbtx, complete); - } catch (const std::runtime_error& e) { - QMessageBox::critical(nullptr, tr("Sign failed"), e.what()); - send_failure = true; - return; - } - if (err == TransactionError::EXTERNAL_SIGNER_NOT_FOUND) { - //: "External signer" means using devices such as hardware wallets. - QMessageBox::critical(nullptr, tr("External signer not found"), "External signer not found"); - send_failure = true; - return; - } - if (err == TransactionError::EXTERNAL_SIGNER_FAILED) { - //: "External signer" means using devices such as hardware wallets. - QMessageBox::critical(nullptr, tr("External signer failure"), "External signer failure"); - send_failure = true; - return; - } - if (err != TransactionError::OK) { - tfm::format(std::cerr, "Failed to sign PSBT"); - processSendCoinsReturn(WalletModel::TransactionCreationFailed); - send_failure = true; - return; + CMutableTransaction mtx = CMutableTransaction{*(m_current_transaction->getWtx())}; + PartiallySignedTransaction psbtx(mtx); + bool complete = false; + // Always fill without signing first. This prevents an external signer + // from being called prematurely and is not expensive. + TransactionError err = model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, /*n_signed=*/nullptr, psbtx, complete); + assert(!complete); + assert(err == TransactionError::OK); + send_failure = !signWithExternalSigner(psbtx, mtx, complete); + // Don't broadcast when user rejects it on the device or there's a failure: + broadcast = complete && !send_failure; + if (!send_failure) { + // A transaction signed with an external signer is not always complete, + // e.g. in a multisig wallet. + if (complete) { + // Prepare transaction for broadcast transaction if complete + const CTransactionRef tx = MakeTransactionRef(mtx); + m_current_transaction->setWtx(tx); + } else { + presentPSBT(psbtx); + } } - // fillPSBT does not always properly finalize - complete = FinalizeAndExtractPSBT(psbtx, mtx); } - // Broadcast transaction if complete (even with an external signer this - // is not always the case, e.g. in a multisig wallet). - if (complete) { - const CTransactionRef tx = MakeTransactionRef(mtx); - m_current_transaction->setWtx(tx); + // Broadcast the transaction, unless an external signer was used and it + // failed, or more signatures are needed. + if (broadcast) { + // now send the prepared transaction WalletModel::SendCoinsReturn sendStatus = model->sendCoins(*m_current_transaction); // process sendStatus and on error generate message shown to user processSendCoinsReturn(sendStatus); @@ -476,64 +552,6 @@ void SendCoinsDialog::sendButtonClicked([[maybe_unused]] bool checked) } else { send_failure = true; } - return; - } - - // Copy PSBT to clipboard and offer to save - assert(!complete); - // Serialize the PSBT - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); - ssTx << psbtx; - GUIUtil::setClipboard(EncodeBase64(ssTx.str()).c_str()); - QMessageBox msgBox; - msgBox.setText("Unsigned Transaction"); - msgBox.setInformativeText("The PSBT has been copied to the clipboard. You can also save it."); - msgBox.setStandardButtons(QMessageBox::Save | QMessageBox::Discard); - msgBox.setDefaultButton(QMessageBox::Discard); - switch (msgBox.exec()) { - case QMessageBox::Save: { - QString selectedFilter; - QString fileNameSuggestion = ""; - bool first = true; - for (const SendCoinsRecipient &rcp : m_current_transaction->getRecipients()) { - if (!first) { - fileNameSuggestion.append(" - "); - } - QString labelOrAddress = rcp.label.isEmpty() ? rcp.address : rcp.label; - QString amount = BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), rcp.amount); - fileNameSuggestion.append(labelOrAddress + "-" + amount); - first = false; - } - fileNameSuggestion.append(".psbt"); - QString filename = GUIUtil::getSaveFileName(this, - tr("Save Transaction Data"), fileNameSuggestion, - //: Expanded name of the binary PSBT file format. See: BIP 174. - tr("Partially Signed Transaction (Binary)") + QLatin1String(" (*.psbt)"), &selectedFilter); - if (filename.isEmpty()) { - return; - } - std::ofstream out{filename.toLocal8Bit().data(), std::ofstream::out | std::ofstream::binary}; - out << ssTx.str(); - out.close(); - Q_EMIT message(tr("PSBT saved"), "PSBT saved to disk", CClientUIInterface::MSG_INFORMATION); - break; - } - case QMessageBox::Discard: - break; - default: - assert(false); - } // msgBox.exec() - } else { - assert(!model->wallet().privateKeysDisabled()); - // now send the prepared transaction - WalletModel::SendCoinsReturn sendStatus = model->sendCoins(*m_current_transaction); - // process sendStatus and on error generate message shown to user - processSendCoinsReturn(sendStatus); - - if (sendStatus.status == WalletModel::OK) { - Q_EMIT coinsSent(m_current_transaction->getWtx()->GetHash()); - } else { - send_failure = true; } } if (!send_failure) { diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index 4a16702756..400503d0c0 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -70,6 +70,8 @@ private: bool fFeeMinimized; const PlatformStyle *platformStyle; + // Copy PSBT to clipboard and offer to save it. + void presentPSBT(PartiallySignedTransaction& psbt); // Process WalletModel::SendCoinsReturn and generate a pair consisting // of a message and message flags for use in Q_EMIT message(). // Additional parameter msgArg can be used via .arg(msgArg). @@ -77,6 +79,15 @@ private: void minimizeFeeSection(bool fMinimize); // Format confirmation message bool PrepareSendText(QString& question_string, QString& informative_text, QString& detailed_text); + /* Sign PSBT using external signer. + * + * @param[in,out] psbtx the PSBT to sign + * @param[in,out] mtx needed to attempt to finalize + * @param[in,out] complete whether the PSBT is complete (a successfully signed multisig transaction may not be complete) + * + * @returns false if any failure occurred, which may include the user rejection of a transaction on the device. + */ + bool signWithExternalSigner(PartiallySignedTransaction& psbt, CMutableTransaction& mtx, bool& complete); void updateFeeMinimizedLabel(); void updateCoinControlState(); @@ -117,6 +128,8 @@ class SendConfirmationDialog : public QMessageBox public: SendConfirmationDialog(const QString& title, const QString& text, const QString& informative_text = "", const QString& detailed_text = "", int secDelay = SEND_CONFIRM_DELAY, bool enable_send = true, bool always_show_unsigned = true, QWidget* parent = nullptr); + /* Returns QMessageBox::Cancel, QMessageBox::Yes when "Send" is + clicked and QMessageBox::Save when "Create Unsigned" is clicked. */ int exec() override; private Q_SLOTS: diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index 74bedbf020..1f4b30534b 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -91,7 +91,7 @@ void SignVerifyMessageDialog::on_addressBookButton_SM_clicked() { if (model && model->getAddressTableModel()) { - model->refresh(/* pk_hash_only */ true); + model->refresh(/*pk_hash_only=*/true); AddressBookPage dlg(platformStyle, AddressBookPage::ForSelection, AddressBookPage::ReceivingTab, this); dlg.setModel(model->getAddressTableModel()); if (dlg.exec()) diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 66637a5dcf..ba0e4c686f 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -8,6 +8,7 @@ #include <interfaces/chain.h> #include <interfaces/node.h> +#include <qt/addressbookpage.h> #include <qt/clientmodel.h> #include <qt/editaddressdialog.h> #include <qt/optionsmodel.h> @@ -23,8 +24,9 @@ #include <chrono> #include <QApplication> -#include <QTimer> #include <QMessageBox> +#include <QTableView> +#include <QTimer> using wallet::AddWallet; using wallet::CWallet; @@ -131,14 +133,19 @@ void TestAddAddressesToSendBook(interfaces::Node& node) EditAddressDialog editAddressDialog(EditAddressDialog::NewSendingAddress); editAddressDialog.setModel(walletModel.getAddressTableModel()); + AddressBookPage address_book{platformStyle.get(), AddressBookPage::ForEditing, AddressBookPage::SendingTab}; + address_book.setModel(walletModel.getAddressTableModel()); + auto table_view = address_book.findChild<QTableView*>("tableView"); + QCOMPARE(table_view->model()->rowCount(), 1); + EditAddressAndSubmit( &editAddressDialog, QString("uhoh"), preexisting_r_address, QString( "Address \"%1\" already exists as a receiving address with label " "\"%2\" and so cannot be added as a sending address." ).arg(preexisting_r_address).arg(r_label)); - check_addbook_size(2); + QCOMPARE(table_view->model()->rowCount(), 1); EditAddressAndSubmit( &editAddressDialog, QString("uhoh, different"), preexisting_s_address, @@ -146,15 +153,15 @@ void TestAddAddressesToSendBook(interfaces::Node& node) "The entered address \"%1\" is already in the address book with " "label \"%2\"." ).arg(preexisting_s_address).arg(s_label)); - check_addbook_size(2); + QCOMPARE(table_view->model()->rowCount(), 1); // Submit a new address which should add successfully - we expect the // warning message to be blank. EditAddressAndSubmit( &editAddressDialog, QString("new"), new_address, QString("")); - check_addbook_size(3); + QCOMPARE(table_view->model()->rowCount(), 2); } } // namespace diff --git a/src/qt/test/optiontests.cpp b/src/qt/test/optiontests.cpp new file mode 100644 index 0000000000..4a943a6343 --- /dev/null +++ b/src/qt/test/optiontests.cpp @@ -0,0 +1,68 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <init.h> +#include <qt/bitcoin.h> +#include <qt/test/optiontests.h> +#include <test/util/setup_common.h> +#include <util/system.h> + +#include <QSettings> +#include <QTest> + +#include <univalue.h> + +//! Entry point for BitcoinApplication tests. +void OptionTests::optionTests() +{ + // Test regression https://github.com/bitcoin/bitcoin/issues/24457. Ensure + // that setting integer prune value doesn't cause an exception to be thrown + // in the OptionsModel constructor + gArgs.LockSettings([&](util::Settings& settings) { + settings.forced_settings.erase("prune"); + settings.rw_settings["prune"] = 3814; + }); + gArgs.WriteSettingsFile(); + OptionsModel{}; + gArgs.LockSettings([&](util::Settings& settings) { + settings.rw_settings.erase("prune"); + }); + gArgs.WriteSettingsFile(); +} + +void OptionTests::parametersInteraction() +{ + // Test that the bug https://github.com/bitcoin-core/gui/issues/567 does not resurface. + // It was fixed via https://github.com/bitcoin-core/gui/pull/568. + // With fListen=false in ~/.config/Bitcoin/Bitcoin-Qt.conf and all else left as default, + // bitcoin-qt should set both -listen and -listenonion to false and start successfully. + gArgs.ClearPathCache(); + + gArgs.LockSettings([&](util::Settings& s) { + s.forced_settings.erase("listen"); + s.forced_settings.erase("listenonion"); + }); + QVERIFY(!gArgs.IsArgSet("-listen")); + QVERIFY(!gArgs.IsArgSet("-listenonion")); + + QSettings settings; + settings.setValue("fListen", false); + + OptionsModel{}; + + const bool expected{false}; + + QVERIFY(gArgs.IsArgSet("-listen")); + QCOMPARE(gArgs.GetBoolArg("-listen", !expected), expected); + + QVERIFY(gArgs.IsArgSet("-listenonion")); + QCOMPARE(gArgs.GetBoolArg("-listenonion", !expected), expected); + + QVERIFY(AppInitParameterInteraction(gArgs)); + + // cleanup + settings.remove("fListen"); + QVERIFY(!settings.contains("fListen")); + gArgs.ClearPathCache(); +} diff --git a/src/qt/test/optiontests.h b/src/qt/test/optiontests.h new file mode 100644 index 0000000000..39c1612c8f --- /dev/null +++ b/src/qt/test/optiontests.h @@ -0,0 +1,26 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_QT_TEST_OPTIONTESTS_H +#define BITCOIN_QT_TEST_OPTIONTESTS_H + +#include <qt/optionsmodel.h> + +#include <QObject> + +class OptionTests : public QObject +{ + Q_OBJECT +public: + explicit OptionTests(interfaces::Node& node) : m_node(node) {} + +private Q_SLOTS: + void optionTests(); + void parametersInteraction(); + +private: + interfaces::Node& m_node; +}; + +#endif // BITCOIN_QT_TEST_OPTIONTESTS_H diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index 10b7e2ffe7..aeedd92834 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -10,6 +10,7 @@ #include <interfaces/node.h> #include <qt/bitcoin.h> #include <qt/test/apptests.h> +#include <qt/test/optiontests.h> #include <qt/test/rpcnestedtests.h> #include <qt/test/uritests.h> #include <test/util/setup_common.h> @@ -20,8 +21,10 @@ #endif // ENABLE_WALLET #include <QApplication> +#include <QDebug> #include <QObject> #include <QTest> + #include <functional> #if defined(QT_STATICPLUGIN) @@ -68,8 +71,6 @@ int main(int argc, char* argv[]) gArgs.ForceSetArg("-upnp", "0"); gArgs.ForceSetArg("-natpmp", "0"); - bool fInvalid = false; - // Prefer the "minimal" platform for the test instead of the normal default // platform ("xcb", "windows", or "cocoa") so tests can't unintentionally // interfere with any background GUIs and don't require extra resources. @@ -85,28 +86,32 @@ int main(int argc, char* argv[]) app.setApplicationName("Bitcoin-Qt-test"); app.createNode(*init); + int num_test_failures{0}; + AppTests app_tests(app); - if (QTest::qExec(&app_tests) != 0) { - fInvalid = true; - } + num_test_failures += QTest::qExec(&app_tests); + + OptionTests options_tests(app.node()); + num_test_failures += QTest::qExec(&options_tests); + URITests test1; - if (QTest::qExec(&test1) != 0) { - fInvalid = true; - } + num_test_failures += QTest::qExec(&test1); + RPCNestedTests test3(app.node()); - if (QTest::qExec(&test3) != 0) { - fInvalid = true; - } + num_test_failures += QTest::qExec(&test3); + #ifdef ENABLE_WALLET WalletTests test5(app.node()); - if (QTest::qExec(&test5) != 0) { - fInvalid = true; - } + num_test_failures += QTest::qExec(&test5); + AddressBookTests test6(app.node()); - if (QTest::qExec(&test6) != 0) { - fInvalid = true; - } + num_test_failures += QTest::qExec(&test6); #endif - return fInvalid; + if (num_test_failures) { + qWarning("\nFailed tests: %d\n", num_test_failures); + } else { + qDebug("\nAll tests passed.\n"); + } + return num_test_failures; } diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 6ab534764b..c4cd0f4cd1 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -7,24 +7,25 @@ #include <interfaces/chain.h> #include <interfaces/node.h> +#include <key_io.h> #include <qt/bitcoinamountfield.h> +#include <qt/bitcoinunits.h> #include <qt/clientmodel.h> #include <qt/optionsmodel.h> +#include <qt/overviewpage.h> #include <qt/platformstyle.h> #include <qt/qvalidatedlineedit.h> +#include <qt/receivecoinsdialog.h> +#include <qt/receiverequestdialog.h> +#include <qt/recentrequeststablemodel.h> #include <qt/sendcoinsdialog.h> #include <qt/sendcoinsentry.h> #include <qt/transactiontablemodel.h> #include <qt/transactionview.h> #include <qt/walletmodel.h> -#include <key_io.h> #include <test/util/setup_common.h> #include <validation.h> #include <wallet/wallet.h> -#include <qt/overviewpage.h> -#include <qt/receivecoinsdialog.h> -#include <qt/recentrequeststablemodel.h> -#include <qt/receiverequestdialog.h> #include <chrono> #include <memory> @@ -196,7 +197,7 @@ void TestGUI(interfaces::Node& node) // Check balance in send dialog QLabel* balanceLabel = sendCoinsDialog.findChild<QLabel*>("labelBalance"); QString balanceText = balanceLabel->text(); - int unit = walletModel.getOptionsModel()->getDisplayUnit(); + BitcoinUnit unit = walletModel.getOptionsModel()->getDisplayUnit(); CAmount balance = walletModel.wallet().getBalance(); QString balanceComparison = BitcoinUnits::formatWithUnit(unit, balance, false, BitcoinUnits::SeparatorStyle::ALWAYS); QCOMPARE(balanceText, balanceComparison); @@ -222,7 +223,7 @@ void TestGUI(interfaces::Node& node) overviewPage.setWalletModel(&walletModel); QLabel* balanceLabel = overviewPage.findChild<QLabel*>("labelBalance"); QString balanceText = balanceLabel->text().trimmed(); - int unit = walletModel.getOptionsModel()->getDisplayUnit(); + BitcoinUnit unit = walletModel.getOptionsModel()->getDisplayUnit(); CAmount balance = walletModel.wallet().getBalance(); QString balanceComparison = BitcoinUnits::formatWithUnit(unit, balance, false, BitcoinUnits::SeparatorStyle::ALWAYS); QCOMPARE(balanceText, balanceComparison); diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index be5851d627..9e92f89543 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -32,20 +32,18 @@ using wallet::ISMINE_SPENDABLE; using wallet::ISMINE_WATCH_ONLY; using wallet::isminetype; -QString TransactionDesc::FormatTxStatus(const interfaces::WalletTx& wtx, const interfaces::WalletTxStatus& status, bool inMempool, int numBlocks) +QString TransactionDesc::FormatTxStatus(const interfaces::WalletTxStatus& status, bool inMempool) { - { - int nDepth = status.depth_in_main_chain; - if (nDepth < 0) { - return tr("conflicted with a transaction with %1 confirmations").arg(-nDepth); - } else if (nDepth == 0) { - const QString abandoned{status.is_abandoned ? QLatin1String(", ") + tr("abandoned") : QString()}; - return tr("0/unconfirmed, %1").arg(inMempool ? tr("in memory pool") : tr("not in memory pool")) + abandoned; - } else if (nDepth < 6) { - return tr("%1/unconfirmed").arg(nDepth); - } else { - return tr("%1 confirmations").arg(nDepth); - } + int depth = status.depth_in_main_chain; + if (depth < 0) { + return tr("conflicted with a transaction with %1 confirmations").arg(-depth); + } else if (depth == 0) { + const QString abandoned{status.is_abandoned ? QLatin1String(", ") + tr("abandoned") : QString()}; + return tr("0/unconfirmed, %1").arg(inMempool ? tr("in memory pool") : tr("not in memory pool")) + abandoned; + } else if (depth < 6) { + return tr("%1/unconfirmed").arg(depth); + } else { + return tr("%1 confirmations").arg(depth); } } @@ -77,7 +75,7 @@ bool GetPaymentRequestMerchant(const std::string& pr, QString& merchant) return false; } -QString TransactionDesc::toHTML(interfaces::Node& node, interfaces::Wallet& wallet, TransactionRecord *rec, int unit) +QString TransactionDesc::toHTML(interfaces::Node& node, interfaces::Wallet& wallet, TransactionRecord* rec, BitcoinUnit unit) { int numBlocks; interfaces::WalletTxStatus status; @@ -95,7 +93,7 @@ QString TransactionDesc::toHTML(interfaces::Node& node, interfaces::Wallet& wall CAmount nDebit = wtx.debit; CAmount nNet = nCredit - nDebit; - strHTML += "<b>" + tr("Status") + ":</b> " + FormatTxStatus(wtx, status, inMempool, numBlocks); + strHTML += "<b>" + tr("Status") + ":</b> " + FormatTxStatus(status, inMempool); strHTML += "<br>"; strHTML += "<b>" + tr("Date") + ":</b> " + (nTime ? GUIUtil::dateTimeStr(nTime) : "") + "<br>"; diff --git a/src/qt/transactiondesc.h b/src/qt/transactiondesc.h index cf955a433c..803e41b699 100644 --- a/src/qt/transactiondesc.h +++ b/src/qt/transactiondesc.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_QT_TRANSACTIONDESC_H #define BITCOIN_QT_TRANSACTIONDESC_H +#include <qt/bitcoinunits.h> + #include <QObject> #include <QString> @@ -24,12 +26,12 @@ class TransactionDesc: public QObject Q_OBJECT public: - static QString toHTML(interfaces::Node& node, interfaces::Wallet& wallet, TransactionRecord *rec, int unit); + static QString toHTML(interfaces::Node& node, interfaces::Wallet& wallet, TransactionRecord* rec, BitcoinUnit unit); private: TransactionDesc() {} - static QString FormatTxStatus(const interfaces::WalletTx& wtx, const interfaces::WalletTxStatus& status, bool inMempool, int numBlocks); + static QString FormatTxStatus(const interfaces::WalletTxStatus& status, bool inMempool); }; #endif // BITCOIN_QT_TRANSACTIONDESC_H diff --git a/src/qt/transactionrecord.h b/src/qt/transactionrecord.h index dd34656d5f..d8748d7dc9 100644 --- a/src/qt/transactionrecord.h +++ b/src/qt/transactionrecord.h @@ -20,13 +20,7 @@ struct WalletTxStatus; /** UI model for transaction status. The transaction status is the part of a transaction that will change over time. */ -class TransactionStatus -{ -public: - TransactionStatus() : countsForBalance(false), sortKey(""), - matures_in(0), status(Unconfirmed), depth(0), open_for(0) - { } - +struct TransactionStatus { enum Status { Confirmed, /**< Have 6 or more confirmations (normal tx) or fully mature (mined tx) **/ /// Normal (sent/received) transactions @@ -40,28 +34,25 @@ public: }; /// Transaction counts towards available balance - bool countsForBalance; + bool countsForBalance{false}; /// Sorting key based on status std::string sortKey; /** @name Generated (mined) transactions @{*/ - int matures_in; + int matures_in{0}; /**@}*/ /** @name Reported status @{*/ - Status status; - qint64 depth; - qint64 open_for; /**< Timestamp if status==OpenUntilDate, otherwise number - of additional blocks that need to be mined before - finalization */ + Status status{Unconfirmed}; + qint64 depth{0}; /**@}*/ /** Current block hash (to know whether cached status is still valid) */ uint256 m_cur_block_hash{}; - bool needsUpdate; + bool needsUpdate{false}; }; /** UI model for a transaction. A core transaction can be represented by multiple UI transactions if it has diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 44b4fee2e7..7b932890cf 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -5,6 +5,7 @@ #include <qt/transactiontablemodel.h> #include <qt/addresstablemodel.h> +#include <qt/bitcoinunits.h> #include <qt/clientmodel.h> #include <qt/guiconstants.h> #include <qt/guiutil.h> @@ -32,11 +33,11 @@ // Amount column is right-aligned it contains numbers static int column_alignments[] = { - Qt::AlignLeft|Qt::AlignVCenter, /* status */ - Qt::AlignLeft|Qt::AlignVCenter, /* watchonly */ - Qt::AlignLeft|Qt::AlignVCenter, /* date */ - Qt::AlignLeft|Qt::AlignVCenter, /* type */ - Qt::AlignLeft|Qt::AlignVCenter, /* address */ + Qt::AlignLeft|Qt::AlignVCenter, /*status=*/ + Qt::AlignLeft|Qt::AlignVCenter, /*watchonly=*/ + Qt::AlignLeft|Qt::AlignVCenter, /*date=*/ + Qt::AlignLeft|Qt::AlignVCenter, /*type=*/ + Qt::AlignLeft|Qt::AlignVCenter, /*address=*/ Qt::AlignRight|Qt::AlignVCenter /* amount */ }; @@ -232,7 +233,7 @@ public: return nullptr; } - QString describe(interfaces::Node& node, interfaces::Wallet& wallet, TransactionRecord *rec, int unit) + QString describe(interfaces::Node& node, interfaces::Wallet& wallet, TransactionRecord* rec, BitcoinUnit unit) { return TransactionDesc::toHTML(node, wallet, rec, unit); } diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 778ef04b77..47f3ba7e7f 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -550,7 +550,7 @@ void TransactionView::openThirdPartyTxUrl(QString url) QWidget *TransactionView::createDateRangeWidget() { dateRangeWidget = new QFrame(); - dateRangeWidget->setFrameStyle(QFrame::Panel | QFrame::Raised); + dateRangeWidget->setFrameStyle(static_cast<int>(QFrame::Panel) | static_cast<int>(QFrame::Raised)); dateRangeWidget->setContentsMargins(1,1,1,1); QHBoxLayout *layout = new QHBoxLayout(dateRangeWidget); layout->setContentsMargins(0,0,0,0); diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index b025bb367c..d27ddf1aba 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -24,6 +24,7 @@ #include <QApplication> #include <QMessageBox> +#include <QMetaObject> #include <QMutexLocker> #include <QThread> #include <QTimer> @@ -135,7 +136,7 @@ WalletModel* WalletController::getOrCreateWallet(std::unique_ptr<interfaces::Wal // handled on the GUI event loop. wallet_model->moveToThread(thread()); // setParent(parent) must be called in the thread which created the parent object. More details in #18948. - GUIUtil::ObjectInvoke(this, [wallet_model, this] { + QMetaObject::invokeMethod(this, [wallet_model, this] { wallet_model->setParent(this); }, GUIUtil::blockingGUIThreadConnection()); diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index 91ce420a33..dc4e25a02b 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -194,16 +194,16 @@ void WalletFrame::gotoVerifyMessageTab(QString addr) void WalletFrame::gotoLoadPSBT(bool from_clipboard) { - std::string data; + std::vector<unsigned char> data; if (from_clipboard) { std::string raw = QApplication::clipboard()->text().toStdString(); - bool invalid; - data = DecodeBase64(raw, &invalid); - if (invalid) { + auto result = DecodeBase64(raw); + if (!result) { Q_EMIT message(tr("Error"), tr("Unable to decode PSBT from clipboard (invalid base64)"), CClientUIInterface::MSG_ERROR); return; } + data = std::move(*result); } else { QString filename = GUIUtil::getOpenFileName(this, tr("Load Transaction Data"), QString(), @@ -214,12 +214,12 @@ void WalletFrame::gotoLoadPSBT(bool from_clipboard) return; } std::ifstream in{filename.toLocal8Bit().data(), std::ios::binary}; - data = std::string(std::istreambuf_iterator<char>{in}, {}); + data.assign(std::istream_iterator<unsigned char>{in}, {}); } std::string error; PartiallySignedTransaction psbtx; - if (!DecodeRawPSBT(psbtx, data, error)) { + if (!DecodeRawPSBT(psbtx, MakeByteSpan(data), error)) { Q_EMIT message(tr("Error"), tr("Unable to decode PSBT") + "\n" + QString::fromStdString(error), CClientUIInterface::MSG_ERROR); return; } diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 5ee32e79d5..1f6c90af4a 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -145,7 +145,7 @@ void WalletModel::updateWatchOnlyFlag(bool fHaveWatchonly) Q_EMIT notifyWatchonlyChanged(fHaveWatchonly); } -bool WalletModel::validateAddress(const QString &address) +bool WalletModel::validateAddress(const QString& address) const { return IsValidDestinationString(address.toStdString()); } @@ -284,22 +284,22 @@ WalletModel::SendCoinsReturn WalletModel::sendCoins(WalletModelTransaction &tran return SendCoinsReturn(OK); } -OptionsModel *WalletModel::getOptionsModel() +OptionsModel* WalletModel::getOptionsModel() const { return optionsModel; } -AddressTableModel *WalletModel::getAddressTableModel() +AddressTableModel* WalletModel::getAddressTableModel() const { return addressTableModel; } -TransactionTableModel *WalletModel::getTransactionTableModel() +TransactionTableModel* WalletModel::getTransactionTableModel() const { return transactionTableModel; } -RecentRequestsTableModel *WalletModel::getRecentRequestsTableModel() +RecentRequestsTableModel* WalletModel::getRecentRequestsTableModel() const { return recentRequestsTableModel; } @@ -369,7 +369,7 @@ static void NotifyAddressBookChanged(WalletModel *walletmodel, QString strPurpose = QString::fromStdString(purpose); qDebug() << "NotifyAddressBookChanged: " + strAddress + " " + strLabel + " isMine=" + QString::number(isMine) + " purpose=" + strPurpose + " status=" + QString::number(status); - bool invoked = QMetaObject::invokeMethod(walletmodel, "updateAddressBook", Qt::QueuedConnection, + bool invoked = QMetaObject::invokeMethod(walletmodel, "updateAddressBook", Q_ARG(QString, strAddress), Q_ARG(QString, strLabel), Q_ARG(bool, isMine), @@ -557,7 +557,7 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) return true; } -bool WalletModel::displayAddress(std::string sAddress) +bool WalletModel::displayAddress(std::string sAddress) const { CTxDestination dest = DecodeDestination(sAddress); bool res = false; @@ -585,7 +585,7 @@ QString WalletModel::getDisplayName() const return name.isEmpty() ? "["+tr("default wallet")+"]" : name; } -bool WalletModel::isMultiwallet() +bool WalletModel::isMultiwallet() const { return m_node.walletLoader().getWallets().size() > 1; } diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index ad1239ccdc..540fdaafe3 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -77,15 +77,15 @@ public: Unlocked // wallet->IsCrypted() && !wallet->IsLocked() }; - OptionsModel *getOptionsModel(); - AddressTableModel *getAddressTableModel(); - TransactionTableModel *getTransactionTableModel(); - RecentRequestsTableModel *getRecentRequestsTableModel(); + OptionsModel* getOptionsModel() const; + AddressTableModel* getAddressTableModel() const; + TransactionTableModel* getTransactionTableModel() const; + RecentRequestsTableModel* getRecentRequestsTableModel() const; EncryptionStatus getEncryptionStatus() const; // Check address for validity - bool validateAddress(const QString &address); + bool validateAddress(const QString& address) const; // Return status record for SendCoins, contains error id + information struct SendCoinsReturn @@ -137,7 +137,7 @@ public: UnlockContext requestUnlock(); bool bumpFee(uint256 hash, uint256& new_hash); - bool displayAddress(std::string sAddress); + bool displayAddress(std::string sAddress) const; static bool isWalletEnabled(); @@ -149,9 +149,7 @@ public: QString getWalletName() const; QString getDisplayName() const; - bool isMultiwallet(); - - AddressTableModel* getAddressTableModel() const { return addressTableModel; } + bool isMultiwallet() const; void refresh(bool pk_hash_only = false); diff --git a/src/qt/walletview.cpp b/src/qt/walletview.cpp index e7ec54721a..344bf628bb 100644 --- a/src/qt/walletview.cpp +++ b/src/qt/walletview.cpp @@ -23,7 +23,6 @@ #include <util/strencodings.h> #include <QAction> -#include <QActionGroup> #include <QFileDialog> #include <QHBoxLayout> #include <QProgressDialog> diff --git a/src/qt/walletview.h b/src/qt/walletview.h index 2f9d344bc8..301084ffa9 100644 --- a/src/qt/walletview.h +++ b/src/qt/walletview.h @@ -6,6 +6,7 @@ #define BITCOIN_QT_WALLETVIEW_H #include <consensus/amount.h> +#include <qt/bitcoinunits.h> #include <QStackedWidget> @@ -115,7 +116,7 @@ Q_SIGNALS: /** Encryption status of wallet changed */ void encryptionStatusChanged(); /** Notify that a new transaction appeared */ - void incomingTransaction(const QString& date, int unit, const CAmount& amount, const QString& type, const QString& address, const QString& label, const QString& walletName); + void incomingTransaction(const QString& date, BitcoinUnit unit, const CAmount& amount, const QString& type, const QString& address, const QString& label, const QString& walletName); /** Notify that the out of sync warning icon has been pressed */ void outOfSyncWarningClicked(); }; diff --git a/src/random.cpp b/src/random.cpp index b862510524..6ae08103b1 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -13,9 +13,10 @@ #include <compat.h> // for Windows API #include <wincrypt.h> #endif -#include <logging.h> // for LogPrintf() +#include <logging.h> #include <randomenv.h> #include <support/allocators/secure.h> +#include <span.h> #include <sync.h> // for Mutex #include <util/time.h> // for GetTimeMicros() @@ -578,8 +579,8 @@ static void ProcRand(unsigned char* out, int num, RNGLevel level) noexcept } } -void GetRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::FAST); } -void GetStrongRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::SLOW); } +void GetRandBytes(Span<unsigned char> bytes) noexcept { ProcRand(bytes.data(), bytes.size(), RNGLevel::FAST); } +void GetStrongRandBytes(Span<unsigned char> bytes) noexcept { ProcRand(bytes.data(), bytes.size(), RNGLevel::SLOW); } void RandAddPeriodic() noexcept { ProcRand(nullptr, 0, RNGLevel::PERIODIC); } void RandAddEvent(const uint32_t event_info) noexcept { GetRNGState().AddEvent(event_info); } @@ -598,7 +599,7 @@ int GetRandInt(int nMax) noexcept uint256 GetRandHash() noexcept { uint256 hash; - GetRandBytes((unsigned char*)&hash, sizeof(hash)); + GetRandBytes(hash); return hash; } diff --git a/src/random.h b/src/random.h index 97302d61ab..4679a2b40c 100644 --- a/src/random.h +++ b/src/random.h @@ -8,6 +8,7 @@ #include <crypto/chacha20.h> #include <crypto/common.h> +#include <span.h> #include <uint256.h> #include <chrono> @@ -66,7 +67,7 @@ * * Thread-safe. */ -void GetRandBytes(unsigned char* buf, int num) noexcept; +void GetRandBytes(Span<unsigned char> bytes) noexcept; /** Generate a uniform random integer in the range [0..range). Precondition: range > 0 */ uint64_t GetRand(uint64_t nMax) noexcept; /** Generate a uniform random duration in the range [0..max). Precondition: max.count() > 0 */ @@ -105,7 +106,7 @@ uint256 GetRandHash() noexcept; * * Thread-safe. */ -void GetStrongRandBytes(unsigned char* buf, int num) noexcept; +void GetStrongRandBytes(Span<unsigned char> bytes) noexcept; /** * Gather entropy from various expensive sources, and feed them to the PRNG state. @@ -222,6 +223,17 @@ public: /** Generate a random boolean. */ bool randbool() noexcept { return randbits(1); } + /** Return the time point advanced by a uniform random duration. */ + template <typename Tp> + Tp rand_uniform_delay(const Tp& time, typename Tp::duration range) + { + using Dur = typename Tp::duration; + Dur dur{range.count() > 0 ? /* interval [0..range) */ Dur{randrange(range.count())} : + range.count() < 0 ? /* interval (range..0] */ -Dur{randrange(-range.count())} : + /* interval [0..0] */ Dur{0}}; + return time + dur; + } + // Compatibility with the C++11 UniformRandomBitGenerator concept typedef uint64_t result_type; static constexpr uint64_t min() { return 0; } diff --git a/src/randomenv.cpp b/src/randomenv.cpp index 388841289a..c5dca346d6 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -57,8 +57,7 @@ #include <sys/auxv.h> #endif -//! Necessary on some platforms -extern char** environ; +extern char** environ; // NOLINT(readability-redundant-declaration): Necessary on some platforms namespace { @@ -363,10 +362,10 @@ void RandAddStaticEnv(CSHA512& hasher) #if HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS // Network interfaces - struct ifaddrs *ifad = NULL; + struct ifaddrs *ifad = nullptr; getifaddrs(&ifad); struct ifaddrs *ifit = ifad; - while (ifit != NULL) { + while (ifit != nullptr) { hasher.Write((const unsigned char*)&ifit, sizeof(ifit)); hasher.Write((const unsigned char*)ifit->ifa_name, strlen(ifit->ifa_name) + 1); hasher.Write((const unsigned char*)&ifit->ifa_flags, sizeof(ifit->ifa_flags)); diff --git a/src/rest.cpp b/src/rest.cpp index 063872b47a..22b5d2e074 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -3,6 +3,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <rest.h> + #include <blockfilter.h> #include <chain.h> #include <chainparams.h> @@ -15,6 +17,7 @@ #include <primitives/block.h> #include <primitives/transaction.h> #include <rpc/blockchain.h> +#include <rpc/mempool.h> #include <rpc/protocol.h> #include <rpc/server.h> #include <rpc/server_util.h> @@ -27,34 +30,25 @@ #include <version.h> #include <any> - -#include <boost/algorithm/string.hpp> +#include <string> #include <univalue.h> using node::GetTransaction; -using node::IsBlockPruned; using node::NodeContext; using node::ReadBlockFromDisk; static const size_t MAX_GETUTXOS_OUTPOINTS = 15; //allow a max of 15 outpoints to be queried at once static constexpr unsigned int MAX_REST_HEADERS_RESULTS = 2000; -enum class RetFormat { - UNDEF, - BINARY, - HEX, - JSON, -}; - static const struct { - RetFormat rf; + RESTResponseFormat rf; const char* name; } rf_names[] = { - {RetFormat::UNDEF, ""}, - {RetFormat::BINARY, "bin"}, - {RetFormat::HEX, "hex"}, - {RetFormat::JSON, "json"}, + {RESTResponseFormat::UNDEF, ""}, + {RESTResponseFormat::BINARY, "bin"}, + {RESTResponseFormat::HEX, "hex"}, + {RESTResponseFormat::JSON, "json"}, }; struct CCoin { @@ -137,25 +131,28 @@ static ChainstateManager* GetChainman(const std::any& context, HTTPRequest* req) return node_context->chainman.get(); } -static RetFormat ParseDataFormat(std::string& param, const std::string& strReq) +RESTResponseFormat ParseDataFormat(std::string& param, const std::string& strReq) { - const std::string::size_type pos = strReq.rfind('.'); - if (pos == std::string::npos) - { - param = strReq; + // Remove query string (if any, separated with '?') as it should not interfere with + // parsing param and data format + param = strReq.substr(0, strReq.rfind('?')); + const std::string::size_type pos_format{param.rfind('.')}; + + // No format string is found + if (pos_format == std::string::npos) { return rf_names[0].rf; } - param = strReq.substr(0, pos); - const std::string suff(strReq, pos + 1); - + // Match format string to available formats + const std::string suffix(param, pos_format + 1); for (const auto& rf_name : rf_names) { - if (suff == rf_name.name) + if (suffix == rf_name.name) { + param.erase(pos_format); return rf_name.rf; + } } - /* If no suffix is found, return original string. */ - param = strReq; + // If no suffix is found, return RESTResponseFormat::UNDEF and original string without query string return rf_names[0].rf; } @@ -191,19 +188,28 @@ static bool rest_headers(const std::any& context, if (!CheckWarmup(req)) return false; std::string param; - const RetFormat rf = ParseDataFormat(param, strURIPart); - std::vector<std::string> path; - boost::split(path, param, boost::is_any_of("/")); - - if (path.size() != 2) - return RESTERR(req, HTTP_BAD_REQUEST, "No header count specified. Use /rest/headers/<count>/<hash>.<ext>."); + const RESTResponseFormat rf = ParseDataFormat(param, strURIPart); + std::vector<std::string> path = SplitString(param, '/'); - const auto parsed_count{ToIntegral<size_t>(path[0])}; + std::string raw_count; + std::string hashStr; + if (path.size() == 2) { + // deprecated path: /rest/headers/<count>/<hash> + hashStr = path[1]; + raw_count = path[0]; + } else if (path.size() == 1) { + // new path with query parameter: /rest/headers/<hash>?count=<count> + hashStr = path[0]; + raw_count = req->GetQueryParameter("count").value_or("5"); + } else { + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/headers/<hash>.<ext>?count=<count>"); + } + + const auto parsed_count{ToIntegral<size_t>(raw_count)}; if (!parsed_count.has_value() || *parsed_count < 1 || *parsed_count > MAX_REST_HEADERS_RESULTS) { - return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Header count is invalid or out of acceptable range (1-%u): %s", MAX_REST_HEADERS_RESULTS, path[0])); + return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Header count is invalid or out of acceptable range (1-%u): %s", MAX_REST_HEADERS_RESULTS, raw_count)); } - std::string hashStr = path[1]; uint256 hash; if (!ParseHashStr(hashStr, hash)) return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); @@ -229,7 +235,7 @@ static bool rest_headers(const std::any& context, } switch (rf) { - case RetFormat::BINARY: { + case RESTResponseFormat::BINARY: { CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION); for (const CBlockIndex *pindex : headers) { ssHeader << pindex->GetBlockHeader(); @@ -241,7 +247,7 @@ static bool rest_headers(const std::any& context, return true; } - case RetFormat::HEX: { + case RESTResponseFormat::HEX: { CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION); for (const CBlockIndex *pindex : headers) { ssHeader << pindex->GetBlockHeader(); @@ -252,7 +258,7 @@ static bool rest_headers(const std::any& context, req->WriteReply(HTTP_OK, strHex); return true; } - case RetFormat::JSON: { + case RESTResponseFormat::JSON: { UniValue jsonHeaders(UniValue::VARR); for (const CBlockIndex *pindex : headers) { jsonHeaders.push_back(blockheaderToJSON(tip, pindex)); @@ -276,19 +282,19 @@ static bool rest_block(const std::any& context, if (!CheckWarmup(req)) return false; std::string hashStr; - const RetFormat rf = ParseDataFormat(hashStr, strURIPart); + const RESTResponseFormat rf = ParseDataFormat(hashStr, strURIPart); uint256 hash; if (!ParseHashStr(hashStr, hash)) return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); CBlock block; - CBlockIndex* pblockindex = nullptr; - CBlockIndex* tip = nullptr; + const CBlockIndex* pblockindex = nullptr; + const CBlockIndex* tip = nullptr; + ChainstateManager* maybe_chainman = GetChainman(context, req); + if (!maybe_chainman) return false; + ChainstateManager& chainman = *maybe_chainman; { - ChainstateManager* maybe_chainman = GetChainman(context, req); - if (!maybe_chainman) return false; - ChainstateManager& chainman = *maybe_chainman; LOCK(cs_main); tip = chainman.ActiveChain().Tip(); pblockindex = chainman.m_blockman.LookupBlockIndex(hash); @@ -296,7 +302,7 @@ static bool rest_block(const std::any& context, return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } - if (IsBlockPruned(pblockindex)) + if (chainman.m_blockman.IsBlockPruned(pblockindex)) return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not available (pruned data)"); if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) @@ -304,7 +310,7 @@ static bool rest_block(const std::any& context, } switch (rf) { - case RetFormat::BINARY: { + case RESTResponseFormat::BINARY: { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssBlock << block; std::string binaryBlock = ssBlock.str(); @@ -313,7 +319,7 @@ static bool rest_block(const std::any& context, return true; } - case RetFormat::HEX: { + case RESTResponseFormat::HEX: { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssBlock << block; std::string strHex = HexStr(ssBlock) + "\n"; @@ -322,8 +328,8 @@ static bool rest_block(const std::any& context, return true; } - case RetFormat::JSON: { - UniValue objBlock = blockToJSON(block, tip, pblockindex, tx_verbosity); + case RESTResponseFormat::JSON: { + UniValue objBlock = blockToJSON(chainman.m_blockman, block, tip, pblockindex, tx_verbosity); std::string strJSON = objBlock.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); @@ -351,17 +357,31 @@ static bool rest_filter_header(const std::any& context, HTTPRequest* req, const if (!CheckWarmup(req)) return false; std::string param; - const RetFormat rf = ParseDataFormat(param, strURIPart); - - std::vector<std::string> uri_parts; - boost::split(uri_parts, param, boost::is_any_of("/")); - if (uri_parts.size() != 3) { - return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/blockfilterheaders/<filtertype>/<count>/<blockhash>"); + const RESTResponseFormat rf = ParseDataFormat(param, strURIPart); + + std::vector<std::string> uri_parts = SplitString(param, '/'); + std::string raw_count; + std::string raw_blockhash; + if (uri_parts.size() == 3) { + // deprecated path: /rest/blockfilterheaders/<filtertype>/<count>/<blockhash> + raw_blockhash = uri_parts[2]; + raw_count = uri_parts[1]; + } else if (uri_parts.size() == 2) { + // new path with query parameter: /rest/blockfilterheaders/<filtertype>/<blockhash>?count=<count> + raw_blockhash = uri_parts[1]; + raw_count = req->GetQueryParameter("count").value_or("5"); + } else { + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/blockfilterheaders/<filtertype>/<blockhash>.<ext>?count=<count>"); + } + + const auto parsed_count{ToIntegral<size_t>(raw_count)}; + if (!parsed_count.has_value() || *parsed_count < 1 || *parsed_count > MAX_REST_HEADERS_RESULTS) { + return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Header count is invalid or out of acceptable range (1-%u): %s", MAX_REST_HEADERS_RESULTS, raw_count)); } uint256 block_hash; - if (!ParseHashStr(uri_parts[2], block_hash)) { - return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + uri_parts[2]); + if (!ParseHashStr(raw_blockhash, block_hash)) { + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + raw_blockhash); } BlockFilterType filtertype; @@ -374,11 +394,6 @@ static bool rest_filter_header(const std::any& context, HTTPRequest* req, const return RESTERR(req, HTTP_BAD_REQUEST, "Index is not enabled for filtertype " + uri_parts[0]); } - const auto parsed_count{ToIntegral<size_t>(uri_parts[1])}; - if (!parsed_count.has_value() || *parsed_count < 1 || *parsed_count > MAX_REST_HEADERS_RESULTS) { - return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Header count is invalid or out of acceptable range (1-%u): %s", MAX_REST_HEADERS_RESULTS, uri_parts[1])); - } - std::vector<const CBlockIndex*> headers; headers.reserve(*parsed_count); { @@ -417,7 +432,7 @@ static bool rest_filter_header(const std::any& context, HTTPRequest* req, const } switch (rf) { - case RetFormat::BINARY: { + case RESTResponseFormat::BINARY: { CDataStream ssHeader{SER_NETWORK, PROTOCOL_VERSION}; for (const uint256& header : filter_headers) { ssHeader << header; @@ -428,7 +443,7 @@ static bool rest_filter_header(const std::any& context, HTTPRequest* req, const req->WriteReply(HTTP_OK, binaryHeader); return true; } - case RetFormat::HEX: { + case RESTResponseFormat::HEX: { CDataStream ssHeader{SER_NETWORK, PROTOCOL_VERSION}; for (const uint256& header : filter_headers) { ssHeader << header; @@ -439,7 +454,7 @@ static bool rest_filter_header(const std::any& context, HTTPRequest* req, const req->WriteReply(HTTP_OK, strHex); return true; } - case RetFormat::JSON: { + case RESTResponseFormat::JSON: { UniValue jsonHeaders(UniValue::VARR); for (const uint256& header : filter_headers) { jsonHeaders.push_back(header.GetHex()); @@ -461,11 +476,10 @@ static bool rest_block_filter(const std::any& context, HTTPRequest* req, const s if (!CheckWarmup(req)) return false; std::string param; - const RetFormat rf = ParseDataFormat(param, strURIPart); + const RESTResponseFormat rf = ParseDataFormat(param, strURIPart); // request is sent over URI scheme /rest/blockfilter/filtertype/blockhash - std::vector<std::string> uri_parts; - boost::split(uri_parts, param, boost::is_any_of("/")); + std::vector<std::string> uri_parts = SplitString(param, '/'); if (uri_parts.size() != 2) { return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/blockfilter/<filtertype>/<blockhash>"); } @@ -517,7 +531,7 @@ static bool rest_block_filter(const std::any& context, HTTPRequest* req, const s } switch (rf) { - case RetFormat::BINARY: { + case RESTResponseFormat::BINARY: { CDataStream ssResp{SER_NETWORK, PROTOCOL_VERSION}; ssResp << filter; @@ -526,7 +540,7 @@ static bool rest_block_filter(const std::any& context, HTTPRequest* req, const s req->WriteReply(HTTP_OK, binaryResp); return true; } - case RetFormat::HEX: { + case RESTResponseFormat::HEX: { CDataStream ssResp{SER_NETWORK, PROTOCOL_VERSION}; ssResp << filter; @@ -535,7 +549,7 @@ static bool rest_block_filter(const std::any& context, HTTPRequest* req, const s req->WriteReply(HTTP_OK, strHex); return true; } - case RetFormat::JSON: { + case RESTResponseFormat::JSON: { UniValue ret(UniValue::VOBJ); ret.pushKV("filter", HexStr(filter.GetEncodedFilter())); std::string strJSON = ret.write() + "\n"; @@ -557,10 +571,10 @@ static bool rest_chaininfo(const std::any& context, HTTPRequest* req, const std: if (!CheckWarmup(req)) return false; std::string param; - const RetFormat rf = ParseDataFormat(param, strURIPart); + const RESTResponseFormat rf = ParseDataFormat(param, strURIPart); switch (rf) { - case RetFormat::JSON: { + case RESTResponseFormat::JSON: { JSONRPCRequest jsonRequest; jsonRequest.context = context; jsonRequest.params = UniValue(UniValue::VARR); @@ -583,10 +597,10 @@ static bool rest_mempool_info(const std::any& context, HTTPRequest* req, const s const CTxMemPool* mempool = GetMemPool(context, req); if (!mempool) return false; std::string param; - const RetFormat rf = ParseDataFormat(param, strURIPart); + const RESTResponseFormat rf = ParseDataFormat(param, strURIPart); switch (rf) { - case RetFormat::JSON: { + case RESTResponseFormat::JSON: { UniValue mempoolInfoObject = MempoolInfoToJSON(*mempool); std::string strJSON = mempoolInfoObject.write() + "\n"; @@ -606,10 +620,10 @@ static bool rest_mempool_contents(const std::any& context, HTTPRequest* req, con const CTxMemPool* mempool = GetMemPool(context, req); if (!mempool) return false; std::string param; - const RetFormat rf = ParseDataFormat(param, strURIPart); + const RESTResponseFormat rf = ParseDataFormat(param, strURIPart); switch (rf) { - case RetFormat::JSON: { + case RESTResponseFormat::JSON: { UniValue mempoolObject = MempoolToJSON(*mempool, true); std::string strJSON = mempoolObject.write() + "\n"; @@ -628,7 +642,7 @@ static bool rest_tx(const std::any& context, HTTPRequest* req, const std::string if (!CheckWarmup(req)) return false; std::string hashStr; - const RetFormat rf = ParseDataFormat(hashStr, strURIPart); + const RESTResponseFormat rf = ParseDataFormat(hashStr, strURIPart); uint256 hash; if (!ParseHashStr(hashStr, hash)) @@ -641,13 +655,13 @@ static bool rest_tx(const std::any& context, HTTPRequest* req, const std::string const NodeContext* const node = GetNodeContext(context, req); if (!node) return false; uint256 hashBlock = uint256(); - const CTransactionRef tx = GetTransaction(/* block_index */ nullptr, node->mempool.get(), hash, Params().GetConsensus(), hashBlock); + const CTransactionRef tx = GetTransaction(/*block_index=*/nullptr, node->mempool.get(), hash, Params().GetConsensus(), hashBlock); if (!tx) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } switch (rf) { - case RetFormat::BINARY: { + case RESTResponseFormat::BINARY: { CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssTx << tx; @@ -657,7 +671,7 @@ static bool rest_tx(const std::any& context, HTTPRequest* req, const std::string return true; } - case RetFormat::HEX: { + case RESTResponseFormat::HEX: { CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssTx << tx; @@ -667,9 +681,9 @@ static bool rest_tx(const std::any& context, HTTPRequest* req, const std::string return true; } - case RetFormat::JSON: { + case RESTResponseFormat::JSON: { UniValue objTx(UniValue::VOBJ); - TxToUniv(*tx, hashBlock, objTx); + TxToUniv(*tx, /*block_hash=*/hashBlock, /*entry=*/ objTx); std::string strJSON = objTx.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); @@ -687,13 +701,13 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: if (!CheckWarmup(req)) return false; std::string param; - const RetFormat rf = ParseDataFormat(param, strURIPart); + const RESTResponseFormat rf = ParseDataFormat(param, strURIPart); std::vector<std::string> uriParts; if (param.length() > 1) { std::string strUriParams = param.substr(1); - boost::split(uriParts, strUriParams, boost::is_any_of("/")); + uriParts = SplitString(strUriParams, '/'); } // throw exception in case of an empty request @@ -734,14 +748,14 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: } switch (rf) { - case RetFormat::HEX: { + case RESTResponseFormat::HEX: { // convert hex to bin, continue then with bin part std::vector<unsigned char> strRequestV = ParseHex(strRequestMutable); strRequestMutable.assign(strRequestV.begin(), strRequestV.end()); [[fallthrough]]; } - case RetFormat::BINARY: { + case RESTResponseFormat::BINARY: { try { //deserialize only if user sent a request if (strRequestMutable.size() > 0) @@ -761,7 +775,7 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: break; } - case RetFormat::JSON: { + case RESTResponseFormat::JSON: { if (!fInputParsed) return RESTERR(req, HTTP_BAD_REQUEST, "Error: empty request"); break; @@ -815,7 +829,7 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: } switch (rf) { - case RetFormat::BINARY: { + case RESTResponseFormat::BINARY: { // serialize data // use exact same output as mentioned in Bip64 CDataStream ssGetUTXOResponse(SER_NETWORK, PROTOCOL_VERSION); @@ -827,7 +841,7 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: return true; } - case RetFormat::HEX: { + case RESTResponseFormat::HEX: { CDataStream ssGetUTXOResponse(SER_NETWORK, PROTOCOL_VERSION); ssGetUTXOResponse << chainman.ActiveChain().Height() << chainman.ActiveChain().Tip()->GetBlockHash() << bitmap << outs; std::string strHex = HexStr(ssGetUTXOResponse) + "\n"; @@ -837,7 +851,7 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: return true; } - case RetFormat::JSON: { + case RESTResponseFormat::JSON: { UniValue objGetUTXOResponse(UniValue::VOBJ); // pack in some essentials @@ -854,7 +868,7 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: // include the script in a json output UniValue o(UniValue::VOBJ); - ScriptPubKeyToUniv(coin.out.scriptPubKey, o, true); + ScriptToUniv(coin.out.scriptPubKey, /*out=*/o, /*include_hex=*/true, /*include_address=*/true); utxo.pushKV("scriptPubKey", o); utxos.push_back(utxo); } @@ -877,7 +891,7 @@ static bool rest_blockhash_by_height(const std::any& context, HTTPRequest* req, { if (!CheckWarmup(req)) return false; std::string height_str; - const RetFormat rf = ParseDataFormat(height_str, str_uri_part); + const RESTResponseFormat rf = ParseDataFormat(height_str, str_uri_part); int32_t blockheight = -1; // Initialization done only to prevent valgrind false positive, see https://github.com/bitcoin/bitcoin/pull/18785 if (!ParseInt32(height_str, &blockheight) || blockheight < 0) { @@ -897,19 +911,19 @@ static bool rest_blockhash_by_height(const std::any& context, HTTPRequest* req, pblockindex = active_chain[blockheight]; } switch (rf) { - case RetFormat::BINARY: { + case RESTResponseFormat::BINARY: { CDataStream ss_blockhash(SER_NETWORK, PROTOCOL_VERSION); ss_blockhash << pblockindex->GetBlockHash(); req->WriteHeader("Content-Type", "application/octet-stream"); req->WriteReply(HTTP_OK, ss_blockhash.str()); return true; } - case RetFormat::HEX: { + case RESTResponseFormat::HEX: { req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, pblockindex->GetBlockHash().GetHex() + "\n"); return true; } - case RetFormat::JSON: { + case RESTResponseFormat::JSON: { req->WriteHeader("Content-Type", "application/json"); UniValue resp = UniValue(UniValue::VOBJ); resp.pushKV("blockhash", pblockindex->GetBlockHash().GetHex()); diff --git a/src/rest.h b/src/rest.h new file mode 100644 index 0000000000..49b1c333d0 --- /dev/null +++ b/src/rest.h @@ -0,0 +1,28 @@ +// Copyright (c) 2015-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_REST_H +#define BITCOIN_REST_H + +#include <string> + +enum class RESTResponseFormat { + UNDEF, + BINARY, + HEX, + JSON, +}; + +/** + * Parse a URI to get the data format and URI without data format + * and query string. + * + * @param[out] param The strReq without the data format string and + * without the query string (if any). + * @param[in] strReq The URI to be parsed. + * @return RESTResponseFormat that was parsed from the URI. + */ +RESTResponseFormat ParseDataFormat(std::string& param, const std::string& strReq); + +#endif // BITCOIN_REST_H diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 9817c80cbd..50bf764e53 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -26,10 +26,6 @@ #include <node/coinstats.h> #include <node/context.h> #include <node/utxo_snapshot.h> -#include <policy/feerate.h> -#include <policy/fees.h> -#include <policy/policy.h> -#include <policy/rbf.h> #include <primitives/transaction.h> #include <rpc/server.h> #include <rpc/server_util.h> @@ -40,8 +36,9 @@ #include <txdb.h> #include <txmempool.h> #include <undo.h> +#include <univalue.h> +#include <util/check.h> #include <util/strencodings.h> -#include <util/string.h> #include <util/translation.h> #include <validation.h> #include <validationinterface.h> @@ -50,8 +47,6 @@ #include <stdint.h> -#include <univalue.h> - #include <condition_variable> #include <memory> #include <mutex> @@ -60,7 +55,6 @@ using node::BlockManager; using node::CCoinsStats; using node::CoinStatsHashType; using node::GetUTXOStats; -using node::IsBlockPruned; using node::NodeContext; using node::ReadBlockFromDisk; using node::SnapshotMetadata; @@ -110,7 +104,8 @@ static int ComputeNextBlockAndDepth(const CBlockIndex* tip, const CBlockIndex* b return blockindex == tip ? 1 : -1; } -CBlockIndex* ParseHashOrHeight(const UniValue& param, ChainstateManager& chainman) { +static const CBlockIndex* ParseHashOrHeight(const UniValue& param, ChainstateManager& chainman) +{ LOCK(::cs_main); CChain& active_chain = chainman.ActiveChain(); @@ -127,7 +122,7 @@ CBlockIndex* ParseHashOrHeight(const UniValue& param, ChainstateManager& chainma return active_chain[height]; } else { const uint256 hash{ParseHashV(param, "hash_or_height")}; - CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(hash); + const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(hash); if (!pindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); @@ -166,7 +161,7 @@ UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex return result; } -UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, TxVerbosity verbosity) +UniValue blockToJSON(BlockManager& blockman, const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, TxVerbosity verbosity) { UniValue result = blockheaderToJSON(tip, blockindex); @@ -185,14 +180,14 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIn case TxVerbosity::SHOW_DETAILS: case TxVerbosity::SHOW_DETAILS_AND_PREVOUT: CBlockUndo blockUndo; - const bool have_undo{WITH_LOCK(::cs_main, return !IsBlockPruned(blockindex) && UndoReadFromDisk(blockUndo, blockindex))}; + const bool have_undo{WITH_LOCK(::cs_main, return !blockman.IsBlockPruned(blockindex) && UndoReadFromDisk(blockUndo, blockindex))}; for (size_t i = 0; i < block.vtx.size(); ++i) { const CTransactionRef& tx = block.vtx.at(i); // coinbase transaction (i.e. i == 0) doesn't have undo data const CTxUndo* txundo = (have_undo && i > 0) ? &blockUndo.vtxundo.at(i - 1) : nullptr; UniValue objTx(UniValue::VOBJ); - TxToUniv(*tx, uint256(), objTx, true, RPCSerializationFlags(), txundo, verbosity); + TxToUniv(*tx, /*block_hash=*/uint256(), /*entry=*/objTx, /*include_hex=*/true, RPCSerializationFlags(), txundo, verbosity); txs.push_back(objTx); } break; @@ -426,366 +421,6 @@ static RPCHelpMan getdifficulty() }; } -static std::vector<RPCResult> MempoolEntryDescription() { return { - RPCResult{RPCResult::Type::NUM, "vsize", "virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted."}, - RPCResult{RPCResult::Type::NUM, "weight", "transaction weight as defined in BIP 141."}, - RPCResult{RPCResult::Type::STR_AMOUNT, "fee", /*optional=*/true, - "transaction fee, denominated in " + CURRENCY_UNIT + " (DEPRECATED, returned only if config option -deprecatedrpc=fees is passed)"}, - RPCResult{RPCResult::Type::STR_AMOUNT, "modifiedfee", /*optional=*/true, - "transaction fee with fee deltas used for mining priority, denominated in " + CURRENCY_UNIT + - " (DEPRECATED, returned only if config option -deprecatedrpc=fees is passed)"}, - RPCResult{RPCResult::Type::NUM_TIME, "time", "local time transaction entered pool in seconds since 1 Jan 1970 GMT"}, - RPCResult{RPCResult::Type::NUM, "height", "block height when transaction entered pool"}, - RPCResult{RPCResult::Type::NUM, "descendantcount", "number of in-mempool descendant transactions (including this one)"}, - RPCResult{RPCResult::Type::NUM, "descendantsize", "virtual transaction size of in-mempool descendants (including this one)"}, - RPCResult{RPCResult::Type::STR_AMOUNT, "descendantfees", /*optional=*/true, - "transaction fees of in-mempool descendants (including this one) with fee deltas used for mining priority, denominated in " + - CURRENCY_ATOM + "s (DEPRECATED, returned only if config option -deprecatedrpc=fees is passed)"}, - RPCResult{RPCResult::Type::NUM, "ancestorcount", "number of in-mempool ancestor transactions (including this one)"}, - RPCResult{RPCResult::Type::NUM, "ancestorsize", "virtual transaction size of in-mempool ancestors (including this one)"}, - RPCResult{RPCResult::Type::STR_AMOUNT, "ancestorfees", /*optional=*/true, - "transaction fees of in-mempool ancestors (including this one) with fee deltas used for mining priority, denominated in " + - CURRENCY_ATOM + "s (DEPRECATED, returned only if config option -deprecatedrpc=fees is passed)"}, - RPCResult{RPCResult::Type::STR_HEX, "wtxid", "hash of serialized transaction, including witness data"}, - RPCResult{RPCResult::Type::OBJ, "fees", "", - { - RPCResult{RPCResult::Type::STR_AMOUNT, "base", "transaction fee, denominated in " + CURRENCY_UNIT}, - RPCResult{RPCResult::Type::STR_AMOUNT, "modified", "transaction fee with fee deltas used for mining priority, denominated in " + CURRENCY_UNIT}, - RPCResult{RPCResult::Type::STR_AMOUNT, "ancestor", "transaction fees of in-mempool ancestors (including this one) with fee deltas used for mining priority, denominated in " + CURRENCY_UNIT}, - RPCResult{RPCResult::Type::STR_AMOUNT, "descendant", "transaction fees of in-mempool descendants (including this one) with fee deltas used for mining priority, denominated in " + CURRENCY_UNIT}, - }}, - RPCResult{RPCResult::Type::ARR, "depends", "unconfirmed transactions used as inputs for this transaction", - {RPCResult{RPCResult::Type::STR_HEX, "transactionid", "parent transaction id"}}}, - RPCResult{RPCResult::Type::ARR, "spentby", "unconfirmed transactions spending outputs from this transaction", - {RPCResult{RPCResult::Type::STR_HEX, "transactionid", "child transaction id"}}}, - RPCResult{RPCResult::Type::BOOL, "bip125-replaceable", "Whether this transaction could be replaced due to BIP125 (replace-by-fee)"}, - RPCResult{RPCResult::Type::BOOL, "unbroadcast", "Whether this transaction is currently unbroadcast (initial broadcast not yet acknowledged by any peers)"}, -};} - -static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPoolEntry& e) EXCLUSIVE_LOCKS_REQUIRED(pool.cs) -{ - AssertLockHeld(pool.cs); - - info.pushKV("vsize", (int)e.GetTxSize()); - info.pushKV("weight", (int)e.GetTxWeight()); - // TODO: top-level fee fields are deprecated. deprecated_fee_fields_enabled blocks should be removed in v24 - const bool deprecated_fee_fields_enabled{IsDeprecatedRPCEnabled("fees")}; - if (deprecated_fee_fields_enabled) { - info.pushKV("fee", ValueFromAmount(e.GetFee())); - info.pushKV("modifiedfee", ValueFromAmount(e.GetModifiedFee())); - } - info.pushKV("time", count_seconds(e.GetTime())); - info.pushKV("height", (int)e.GetHeight()); - info.pushKV("descendantcount", e.GetCountWithDescendants()); - info.pushKV("descendantsize", e.GetSizeWithDescendants()); - if (deprecated_fee_fields_enabled) { - info.pushKV("descendantfees", e.GetModFeesWithDescendants()); - } - info.pushKV("ancestorcount", e.GetCountWithAncestors()); - info.pushKV("ancestorsize", e.GetSizeWithAncestors()); - if (deprecated_fee_fields_enabled) { - info.pushKV("ancestorfees", e.GetModFeesWithAncestors()); - } - info.pushKV("wtxid", pool.vTxHashes[e.vTxHashesIdx].first.ToString()); - - UniValue fees(UniValue::VOBJ); - fees.pushKV("base", ValueFromAmount(e.GetFee())); - fees.pushKV("modified", ValueFromAmount(e.GetModifiedFee())); - fees.pushKV("ancestor", ValueFromAmount(e.GetModFeesWithAncestors())); - fees.pushKV("descendant", ValueFromAmount(e.GetModFeesWithDescendants())); - info.pushKV("fees", fees); - - const CTransaction& tx = e.GetTx(); - std::set<std::string> setDepends; - for (const CTxIn& txin : tx.vin) - { - if (pool.exists(GenTxid::Txid(txin.prevout.hash))) - setDepends.insert(txin.prevout.hash.ToString()); - } - - UniValue depends(UniValue::VARR); - for (const std::string& dep : setDepends) - { - depends.push_back(dep); - } - - info.pushKV("depends", depends); - - UniValue spent(UniValue::VARR); - const CTxMemPool::txiter& it = pool.mapTx.find(tx.GetHash()); - const CTxMemPoolEntry::Children& children = it->GetMemPoolChildrenConst(); - for (const CTxMemPoolEntry& child : children) { - spent.push_back(child.GetTx().GetHash().ToString()); - } - - info.pushKV("spentby", spent); - - // Add opt-in RBF status - bool rbfStatus = false; - RBFTransactionState rbfState = IsRBFOptIn(tx, pool); - if (rbfState == RBFTransactionState::UNKNOWN) { - throw JSONRPCError(RPC_MISC_ERROR, "Transaction is not in mempool"); - } else if (rbfState == RBFTransactionState::REPLACEABLE_BIP125) { - rbfStatus = true; - } - - info.pushKV("bip125-replaceable", rbfStatus); - info.pushKV("unbroadcast", pool.IsUnbroadcastTx(tx.GetHash())); -} - -UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose, bool include_mempool_sequence) -{ - if (verbose) { - if (include_mempool_sequence) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Verbose results cannot contain mempool sequence values."); - } - LOCK(pool.cs); - UniValue o(UniValue::VOBJ); - for (const CTxMemPoolEntry& e : pool.mapTx) { - const uint256& hash = e.GetTx().GetHash(); - UniValue info(UniValue::VOBJ); - entryToJSON(pool, info, e); - // Mempool has unique entries so there is no advantage in using - // UniValue::pushKV, which checks if the key already exists in O(N). - // UniValue::__pushKV is used instead which currently is O(1). - o.__pushKV(hash.ToString(), info); - } - return o; - } else { - uint64_t mempool_sequence; - std::vector<uint256> vtxid; - { - LOCK(pool.cs); - pool.queryHashes(vtxid); - mempool_sequence = pool.GetSequence(); - } - UniValue a(UniValue::VARR); - for (const uint256& hash : vtxid) - a.push_back(hash.ToString()); - - if (!include_mempool_sequence) { - return a; - } else { - UniValue o(UniValue::VOBJ); - o.pushKV("txids", a); - o.pushKV("mempool_sequence", mempool_sequence); - return o; - } - } -} - -static RPCHelpMan getrawmempool() -{ - return RPCHelpMan{"getrawmempool", - "\nReturns all transaction ids in memory pool as a json array of string transaction ids.\n" - "\nHint: use getmempoolentry to fetch a specific transaction from the mempool.\n", - { - {"verbose", RPCArg::Type::BOOL, RPCArg::Default{false}, "True for a json object, false for array of transaction ids"}, - {"mempool_sequence", RPCArg::Type::BOOL, RPCArg::Default{false}, "If verbose=false, returns a json object with transaction list and mempool sequence number attached."}, - }, - { - RPCResult{"for verbose = false", - RPCResult::Type::ARR, "", "", - { - {RPCResult::Type::STR_HEX, "", "The transaction id"}, - }}, - RPCResult{"for verbose = true", - RPCResult::Type::OBJ_DYN, "", "", - { - {RPCResult::Type::OBJ, "transactionid", "", MempoolEntryDescription()}, - }}, - RPCResult{"for verbose = false and mempool_sequence = true", - RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::ARR, "txids", "", - { - {RPCResult::Type::STR_HEX, "", "The transaction id"}, - }}, - {RPCResult::Type::NUM, "mempool_sequence", "The mempool sequence value."}, - }}, - }, - RPCExamples{ - HelpExampleCli("getrawmempool", "true") - + HelpExampleRpc("getrawmempool", "true") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - bool fVerbose = false; - if (!request.params[0].isNull()) - fVerbose = request.params[0].get_bool(); - - bool include_mempool_sequence = false; - if (!request.params[1].isNull()) { - include_mempool_sequence = request.params[1].get_bool(); - } - - return MempoolToJSON(EnsureAnyMemPool(request.context), fVerbose, include_mempool_sequence); -}, - }; -} - -static RPCHelpMan getmempoolancestors() -{ - return RPCHelpMan{"getmempoolancestors", - "\nIf txid is in the mempool, returns all in-mempool ancestors.\n", - { - {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id (must be in mempool)"}, - {"verbose", RPCArg::Type::BOOL, RPCArg::Default{false}, "True for a json object, false for array of transaction ids"}, - }, - { - RPCResult{"for verbose = false", - RPCResult::Type::ARR, "", "", - {{RPCResult::Type::STR_HEX, "", "The transaction id of an in-mempool ancestor transaction"}}}, - RPCResult{"for verbose = true", - RPCResult::Type::OBJ_DYN, "", "", - { - {RPCResult::Type::OBJ, "transactionid", "", MempoolEntryDescription()}, - }}, - }, - RPCExamples{ - HelpExampleCli("getmempoolancestors", "\"mytxid\"") - + HelpExampleRpc("getmempoolancestors", "\"mytxid\"") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - bool fVerbose = false; - if (!request.params[1].isNull()) - fVerbose = request.params[1].get_bool(); - - uint256 hash = ParseHashV(request.params[0], "parameter 1"); - - const CTxMemPool& mempool = EnsureAnyMemPool(request.context); - LOCK(mempool.cs); - - CTxMemPool::txiter it = mempool.mapTx.find(hash); - if (it == mempool.mapTx.end()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); - } - - CTxMemPool::setEntries setAncestors; - uint64_t noLimit = std::numeric_limits<uint64_t>::max(); - std::string dummy; - mempool.CalculateMemPoolAncestors(*it, setAncestors, noLimit, noLimit, noLimit, noLimit, dummy, false); - - if (!fVerbose) { - UniValue o(UniValue::VARR); - for (CTxMemPool::txiter ancestorIt : setAncestors) { - o.push_back(ancestorIt->GetTx().GetHash().ToString()); - } - return o; - } else { - UniValue o(UniValue::VOBJ); - for (CTxMemPool::txiter ancestorIt : setAncestors) { - const CTxMemPoolEntry &e = *ancestorIt; - const uint256& _hash = e.GetTx().GetHash(); - UniValue info(UniValue::VOBJ); - entryToJSON(mempool, info, e); - o.pushKV(_hash.ToString(), info); - } - return o; - } -}, - }; -} - -static RPCHelpMan getmempooldescendants() -{ - return RPCHelpMan{"getmempooldescendants", - "\nIf txid is in the mempool, returns all in-mempool descendants.\n", - { - {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id (must be in mempool)"}, - {"verbose", RPCArg::Type::BOOL, RPCArg::Default{false}, "True for a json object, false for array of transaction ids"}, - }, - { - RPCResult{"for verbose = false", - RPCResult::Type::ARR, "", "", - {{RPCResult::Type::STR_HEX, "", "The transaction id of an in-mempool descendant transaction"}}}, - RPCResult{"for verbose = true", - RPCResult::Type::OBJ_DYN, "", "", - { - {RPCResult::Type::OBJ, "transactionid", "", MempoolEntryDescription()}, - }}, - }, - RPCExamples{ - HelpExampleCli("getmempooldescendants", "\"mytxid\"") - + HelpExampleRpc("getmempooldescendants", "\"mytxid\"") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - bool fVerbose = false; - if (!request.params[1].isNull()) - fVerbose = request.params[1].get_bool(); - - uint256 hash = ParseHashV(request.params[0], "parameter 1"); - - const CTxMemPool& mempool = EnsureAnyMemPool(request.context); - LOCK(mempool.cs); - - CTxMemPool::txiter it = mempool.mapTx.find(hash); - if (it == mempool.mapTx.end()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); - } - - CTxMemPool::setEntries setDescendants; - mempool.CalculateDescendants(it, setDescendants); - // CTxMemPool::CalculateDescendants will include the given tx - setDescendants.erase(it); - - if (!fVerbose) { - UniValue o(UniValue::VARR); - for (CTxMemPool::txiter descendantIt : setDescendants) { - o.push_back(descendantIt->GetTx().GetHash().ToString()); - } - - return o; - } else { - UniValue o(UniValue::VOBJ); - for (CTxMemPool::txiter descendantIt : setDescendants) { - const CTxMemPoolEntry &e = *descendantIt; - const uint256& _hash = e.GetTx().GetHash(); - UniValue info(UniValue::VOBJ); - entryToJSON(mempool, info, e); - o.pushKV(_hash.ToString(), info); - } - return o; - } -}, - }; -} - -static RPCHelpMan getmempoolentry() -{ - return RPCHelpMan{"getmempoolentry", - "\nReturns mempool data for given transaction\n", - { - {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id (must be in mempool)"}, - }, - RPCResult{ - RPCResult::Type::OBJ, "", "", MempoolEntryDescription()}, - RPCExamples{ - HelpExampleCli("getmempoolentry", "\"mytxid\"") - + HelpExampleRpc("getmempoolentry", "\"mytxid\"") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - uint256 hash = ParseHashV(request.params[0], "parameter 1"); - - const CTxMemPool& mempool = EnsureAnyMemPool(request.context); - LOCK(mempool.cs); - - CTxMemPool::txiter it = mempool.mapTx.find(hash); - if (it == mempool.mapTx.end()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); - } - - const CTxMemPoolEntry &e = *it; - UniValue info(UniValue::VOBJ); - entryToJSON(mempool, info, e); - return info; -}, - }; -} - static RPCHelpMan getblockfrompeer() { return RPCHelpMan{ @@ -795,7 +430,7 @@ static RPCHelpMan getblockfrompeer() "Subsequent calls for the same block and a new peer will cause the response from the previous peer to be ignored.\n\n" "Returns an empty JSON object if the request was successfully scheduled.", { - {"block_hash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The block hash to try to fetch"}, + {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The block hash to try to fetch"}, {"peer_id", RPCArg::Type::NUM, RPCArg::Optional::NO, "The peer to fetch it from (see getpeerinfo for peer IDs)"}, }, RPCResult{RPCResult::Type::OBJ, "", /*optional=*/false, "", {}}, @@ -809,7 +444,7 @@ static RPCHelpMan getblockfrompeer() ChainstateManager& chainman = EnsureChainman(node); PeerManager& peerman = EnsurePeerman(node); - const uint256& block_hash{ParseHashV(request.params[0], "block_hash")}; + const uint256& block_hash{ParseHashV(request.params[0], "blockhash")}; const NodeId peer_id{request.params[1].get_int64()}; const CBlockIndex* const index = WITH_LOCK(cs_main, return chainman.m_blockman.LookupBlockIndex(block_hash);); @@ -854,7 +489,7 @@ static RPCHelpMan getblockhash() if (nHeight < 0 || nHeight > active_chain.Height()) throw JSONRPCError(RPC_INVALID_PARAMETER, "Block height out of range"); - CBlockIndex* pblockindex = active_chain[nHeight]; + const CBlockIndex* pblockindex = active_chain[nHeight]; return pblockindex->GetBlockHash().GetHex(); }, }; @@ -930,11 +565,11 @@ static RPCHelpMan getblockheader() }; } -static CBlock GetBlockChecked(const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) +static CBlock GetBlockChecked(BlockManager& blockman, const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { AssertLockHeld(::cs_main); CBlock block; - if (IsBlockPruned(pblockindex)) { + if (blockman.IsBlockPruned(pblockindex)) { throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)"); } @@ -948,11 +583,11 @@ static CBlock GetBlockChecked(const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_RE return block; } -static CBlockUndo GetUndoChecked(const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +static CBlockUndo GetUndoChecked(BlockManager& blockman, const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { AssertLockHeld(::cs_main); CBlockUndo blockUndo; - if (IsBlockPruned(pblockindex)) { + if (blockman.IsBlockPruned(pblockindex)) { throw JSONRPCError(RPC_MISC_ERROR, "Undo data not available (pruned data)"); } @@ -1036,7 +671,7 @@ static RPCHelpMan getblock() { {RPCResult::Type::STR, "asm", "The asm"}, {RPCResult::Type::STR, "hex", "The hex"}, - {RPCResult::Type::STR, "address", /* optional */ true, "The Bitcoin address (only if a well-defined address exists)"}, + {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, {RPCResult::Type::STR, "type", "The type (one of: " + GetAllOutputTypes() + ")"}, }}, }}, @@ -1066,8 +701,8 @@ static RPCHelpMan getblock() CBlock block; const CBlockIndex* pblockindex; const CBlockIndex* tip; + ChainstateManager& chainman = EnsureAnyChainman(request.context); { - ChainstateManager& chainman = EnsureAnyChainman(request.context); LOCK(cs_main); pblockindex = chainman.m_blockman.LookupBlockIndex(hash); tip = chainman.ActiveChain().Tip(); @@ -1076,7 +711,7 @@ static RPCHelpMan getblock() throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } - block = GetBlockChecked(pblockindex); + block = GetBlockChecked(chainman.m_blockman, pblockindex); } if (verbosity <= 0) @@ -1096,7 +731,7 @@ static RPCHelpMan getblock() tx_verbosity = TxVerbosity::SHOW_DETAILS_AND_PREVOUT; } - return blockToJSON(block, tip, pblockindex, tx_verbosity); + return blockToJSON(chainman.m_blockman, block, tip, pblockindex, tx_verbosity); }, }; } @@ -1125,14 +760,15 @@ static RPCHelpMan pruneblockchain() CChain& active_chain = active_chainstate.m_chain; int heightParam = request.params[0].get_int(); - if (heightParam < 0) + if (heightParam < 0) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Negative block height."); + } // Height value more than a billion is too high to be a block height, and // too low to be a block time (corresponds to timestamp from Sep 2001). if (heightParam > 1000000000) { // Add a 2 hour buffer to include blocks which might have had old timestamps - CBlockIndex* pindex = active_chain.FindEarliestAtLeast(heightParam - TIMESTAMP_WINDOW, 0); + const CBlockIndex* pindex = active_chain.FindEarliestAtLeast(heightParam - TIMESTAMP_WINDOW, 0); if (!pindex) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Could not find block with at least the specified timestamp."); } @@ -1151,12 +787,10 @@ static RPCHelpMan pruneblockchain() } PruneBlockFilesManual(active_chainstate, height); - const CBlockIndex* block = active_chain.Tip(); - CHECK_NONFATAL(block); - while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) { - block = block->pprev; - } - return uint64_t(block->nHeight); + const CBlockIndex& block{*CHECK_NONFATAL(active_chain.Tip())}; + const CBlockIndex* last_block{active_chainstate.m_blockman.GetFirstStoredBlock(block)}; + + return static_cast<uint64_t>(last_block->nHeight); }, }; } @@ -1226,7 +860,7 @@ static RPCHelpMan gettxoutsetinfo() { UniValue ret(UniValue::VOBJ); - CBlockIndex* pindex{nullptr}; + const CBlockIndex* pindex{nullptr}; const CoinStatsHashType hash_type{request.params[0].isNull() ? CoinStatsHashType::HASH_SERIALIZED : ParseHashType(request.params[0].get_str())}; CCoinsStats stats{hash_type}; stats.index_requested = request.params[2].isNull() || request.params[2].get_bool(); @@ -1391,7 +1025,7 @@ static RPCHelpMan gettxout() } ret.pushKV("value", ValueFromAmount(coin.out.nValue)); UniValue o(UniValue::VOBJ); - ScriptPubKeyToUniv(coin.out.scriptPubKey, o, true); + ScriptToUniv(coin.out.scriptPubKey, /*out=*/o, /*include_hex=*/true, /*include_address=*/true); ret.pushKV("scriptPubKey", o); ret.pushKV("coinbase", (bool)coin.fCoinBase); @@ -1481,7 +1115,7 @@ static void SoftForkDescPushBack(const CBlockIndex* blockindex, UniValue& softfo // BIP9 status bip9.pushKV("status", get_state_name(current_state)); bip9.pushKV("since", g_versionbitscache.StateSinceHeight(blockindex->pprev, consensusParams, id)); - bip9.pushKV("status-next", get_state_name(next_state)); + bip9.pushKV("status_next", get_state_name(next_state)); // BIP9 signalling status, if applicable if (has_signal) { @@ -1527,38 +1161,38 @@ RPCHelpMan getblockchaininfo() { /* TODO: from v24, remove -deprecatedrpc=softforks */ return RPCHelpMan{"getblockchaininfo", - "Returns an object containing various state info regarding blockchain processing.\n", - {}, - RPCResult{ - RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR, "chain", "current network name (main, test, signet, regtest)"}, - {RPCResult::Type::NUM, "blocks", "the height of the most-work fully-validated chain. The genesis block has height 0"}, - {RPCResult::Type::NUM, "headers", "the current number of headers we have validated"}, - {RPCResult::Type::STR, "bestblockhash", "the hash of the currently best block"}, - {RPCResult::Type::NUM, "difficulty", "the current difficulty"}, - {RPCResult::Type::NUM_TIME, "time", "The block time expressed in " + UNIX_EPOCH_TIME}, - {RPCResult::Type::NUM_TIME, "mediantime", "The median block time expressed in " + UNIX_EPOCH_TIME}, - {RPCResult::Type::NUM, "verificationprogress", "estimate of verification progress [0..1]"}, - {RPCResult::Type::BOOL, "initialblockdownload", "(debug information) estimate of whether this node is in Initial Block Download mode"}, - {RPCResult::Type::STR_HEX, "chainwork", "total amount of work in active chain, in hexadecimal"}, - {RPCResult::Type::NUM, "size_on_disk", "the estimated size of the block and undo files on disk"}, - {RPCResult::Type::BOOL, "pruned", "if the blocks are subject to pruning"}, - {RPCResult::Type::NUM, "pruneheight", /*optional=*/true, "lowest-height complete block stored (only present if pruning is enabled)"}, - {RPCResult::Type::BOOL, "automatic_pruning", /*optional=*/true, "whether automatic pruning is enabled (only present if pruning is enabled)"}, - {RPCResult::Type::NUM, "prune_target_size", /*optional=*/true, "the target size used by pruning (only present if automatic pruning is enabled)"}, - {RPCResult::Type::OBJ_DYN, "softforks", "(DEPRECATED, returned only if config option -deprecatedrpc=softforks is passed) status of softforks", - { - {RPCResult::Type::OBJ, "xxxx", "name of the softfork", - RPCHelpForDeployment - }, - }}, - {RPCResult::Type::STR, "warnings", "any network and blockchain warnings"}, - }}, - RPCExamples{ - HelpExampleCli("getblockchaininfo", "") + "Returns an object containing various state info regarding blockchain processing.\n", + {}, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR, "chain", "current network name (main, test, signet, regtest)"}, + {RPCResult::Type::NUM, "blocks", "the height of the most-work fully-validated chain. The genesis block has height 0"}, + {RPCResult::Type::NUM, "headers", "the current number of headers we have validated"}, + {RPCResult::Type::STR, "bestblockhash", "the hash of the currently best block"}, + {RPCResult::Type::NUM, "difficulty", "the current difficulty"}, + {RPCResult::Type::NUM_TIME, "time", "The block time expressed in " + UNIX_EPOCH_TIME}, + {RPCResult::Type::NUM_TIME, "mediantime", "The median block time expressed in " + UNIX_EPOCH_TIME}, + {RPCResult::Type::NUM, "verificationprogress", "estimate of verification progress [0..1]"}, + {RPCResult::Type::BOOL, "initialblockdownload", "(debug information) estimate of whether this node is in Initial Block Download mode"}, + {RPCResult::Type::STR_HEX, "chainwork", "total amount of work in active chain, in hexadecimal"}, + {RPCResult::Type::NUM, "size_on_disk", "the estimated size of the block and undo files on disk"}, + {RPCResult::Type::BOOL, "pruned", "if the blocks are subject to pruning"}, + {RPCResult::Type::NUM, "pruneheight", /*optional=*/true, "lowest-height complete block stored (only present if pruning is enabled)"}, + {RPCResult::Type::BOOL, "automatic_pruning", /*optional=*/true, "whether automatic pruning is enabled (only present if pruning is enabled)"}, + {RPCResult::Type::NUM, "prune_target_size", /*optional=*/true, "the target size used by pruning (only present if automatic pruning is enabled)"}, + {RPCResult::Type::OBJ_DYN, "softforks", /*optional=*/true, "(DEPRECATED, returned only if config option -deprecatedrpc=softforks is passed) status of softforks", + { + {RPCResult::Type::OBJ, "xxxx", "name of the softfork", + RPCHelpForDeployment + }, + }}, + {RPCResult::Type::STR, "warnings", "any network and blockchain warnings"}, + }}, + RPCExamples{ + HelpExampleCli("getblockchaininfo", "") + HelpExampleRpc("getblockchaininfo", "") - }, + }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const ArgsManager& args{EnsureAnyArgsman(request.context)}; @@ -1566,30 +1200,23 @@ RPCHelpMan getblockchaininfo() LOCK(cs_main); CChainState& active_chainstate = chainman.ActiveChainstate(); - const CBlockIndex* tip = active_chainstate.m_chain.Tip(); - CHECK_NONFATAL(tip); - const int height = tip->nHeight; + const CBlockIndex& tip{*CHECK_NONFATAL(active_chainstate.m_chain.Tip())}; + const int height{tip.nHeight}; UniValue obj(UniValue::VOBJ); - obj.pushKV("chain", Params().NetworkIDString()); - obj.pushKV("blocks", height); - obj.pushKV("headers", pindexBestHeader ? pindexBestHeader->nHeight : -1); - obj.pushKV("bestblockhash", tip->GetBlockHash().GetHex()); - obj.pushKV("difficulty", (double)GetDifficulty(tip)); - obj.pushKV("time", (int64_t)tip->nTime); - obj.pushKV("mediantime", (int64_t)tip->GetMedianTimePast()); - obj.pushKV("verificationprogress", GuessVerificationProgress(Params().TxData(), tip)); - obj.pushKV("initialblockdownload", active_chainstate.IsInitialBlockDownload()); - obj.pushKV("chainwork", tip->nChainWork.GetHex()); + obj.pushKV("chain", Params().NetworkIDString()); + obj.pushKV("blocks", height); + obj.pushKV("headers", chainman.m_best_header ? chainman.m_best_header->nHeight : -1); + obj.pushKV("bestblockhash", tip.GetBlockHash().GetHex()); + obj.pushKV("difficulty", GetDifficulty(&tip)); + obj.pushKV("time", tip.GetBlockTime()); + obj.pushKV("mediantime", tip.GetMedianTimePast()); + obj.pushKV("verificationprogress", GuessVerificationProgress(Params().TxData(), &tip)); + obj.pushKV("initialblockdownload", active_chainstate.IsInitialBlockDownload()); + obj.pushKV("chainwork", tip.nChainWork.GetHex()); obj.pushKV("size_on_disk", chainman.m_blockman.CalculateCurrentUsage()); - obj.pushKV("pruned", node::fPruneMode); + obj.pushKV("pruned", node::fPruneMode); if (node::fPruneMode) { - const CBlockIndex* block = tip; - CHECK_NONFATAL(block); - while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) { - block = block->pprev; - } - - obj.pushKV("pruneheight", block->nHeight); + obj.pushKV("pruneheight", chainman.m_blockman.GetFirstStoredBlock(tip)->nHeight); // if 0, execution bypasses the whole if block. bool automatic_pruning{args.GetIntArg("-prune", 0) != 1}; @@ -1601,7 +1228,7 @@ RPCHelpMan getblockchaininfo() if (IsDeprecatedRPCEnabled("softforks")) { const Consensus::Params& consensusParams = Params().GetConsensus(); - obj.pushKV("softforks", DeploymentInfo(tip, consensusParams)); + obj.pushKV("softforks", DeploymentInfo(&tip, consensusParams)); } obj.pushKV("warnings", GetWarnings(false).original); @@ -1623,7 +1250,7 @@ const std::vector<RPCResult> RPCHelpForDeployment{ {RPCResult::Type::NUM, "min_activation_height", "minimum height of blocks for which the rules may be enforced"}, {RPCResult::Type::STR, "status", "status of deployment at specified block (one of \"defined\", \"started\", \"locked_in\", \"active\", \"failed\")"}, {RPCResult::Type::NUM, "since", "height of the first block to which the status applies"}, - {RPCResult::Type::STR, "status-next", "status of deployment at the next block"}, + {RPCResult::Type::STR, "status_next", "status of deployment at the next block"}, {RPCResult::Type::OBJ, "statistics", /*optional=*/true, "numeric statistics about signalling for a softfork (only for \"started\" and \"locked_in\" status)", { {RPCResult::Type::NUM, "period", "the length in blocks of the signalling period"}, @@ -1632,7 +1259,7 @@ const std::vector<RPCResult> RPCHelpForDeployment{ {RPCResult::Type::NUM, "count", "the number of blocks with the version bit set in the current period"}, {RPCResult::Type::BOOL, "possible", /*optional=*/true, "returns false if there are not enough blocks left in this period to pass activation threshold (only for \"started\" status)"}, }}, - {RPCResult::Type::STR, "signalling", "indicates blocks that signalled with a # and blocks that did not with a -"}, + {RPCResult::Type::STR, "signalling", /*optional=*/true, "indicates blocks that signalled with a # and blocks that did not with a -"}, }}, }; @@ -1661,7 +1288,7 @@ static RPCHelpMan getdeploymentinfo() RPCResult::Type::OBJ, "", "", { {RPCResult::Type::STR, "hash", "requested block hash (or tip)"}, {RPCResult::Type::NUM, "height", "requested block height (or tip)"}, - {RPCResult::Type::OBJ, "deployments", "", { + {RPCResult::Type::OBJ_DYN, "deployments", "", { {RPCResult::Type::OBJ, "xxxx", "name of the deployment", RPCHelpForDeployment} }}, } @@ -1675,8 +1302,7 @@ static RPCHelpMan getdeploymentinfo() const CBlockIndex* blockindex; if (request.params[0].isNull()) { - blockindex = active_chainstate.m_chain.Tip(); - CHECK_NONFATAL(blockindex); + blockindex = CHECK_NONFATAL(active_chainstate.m_chain.Tip()); } else { const uint256 hash(ParseHashV(request.params[0], "blockhash")); blockindex = chainman.m_blockman.LookupBlockIndex(hash); @@ -1753,10 +1379,10 @@ static RPCHelpMan getchaintips() std::set<const CBlockIndex*> setOrphans; std::set<const CBlockIndex*> setPrevs; - for (const std::pair<const uint256, CBlockIndex*>& item : chainman.BlockIndex()) { - if (!active_chain.Contains(item.second)) { - setOrphans.insert(item.second); - setPrevs.insert(item.second->pprev); + for (const auto& [_, block_index] : chainman.BlockIndex()) { + if (!active_chain.Contains(&block_index)) { + setOrphans.insert(&block_index); + setPrevs.insert(block_index.pprev); } } @@ -1809,53 +1435,6 @@ static RPCHelpMan getchaintips() }; } -UniValue MempoolInfoToJSON(const CTxMemPool& pool) -{ - // Make sure this call is atomic in the pool. - LOCK(pool.cs); - UniValue ret(UniValue::VOBJ); - ret.pushKV("loaded", pool.IsLoaded()); - ret.pushKV("size", (int64_t)pool.size()); - ret.pushKV("bytes", (int64_t)pool.GetTotalTxSize()); - ret.pushKV("usage", (int64_t)pool.DynamicMemoryUsage()); - ret.pushKV("total_fee", ValueFromAmount(pool.GetTotalFee())); - size_t maxmempool = gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; - ret.pushKV("maxmempool", (int64_t) maxmempool); - ret.pushKV("mempoolminfee", ValueFromAmount(std::max(pool.GetMinFee(maxmempool), ::minRelayTxFee).GetFeePerK())); - ret.pushKV("minrelaytxfee", ValueFromAmount(::minRelayTxFee.GetFeePerK())); - ret.pushKV("unbroadcastcount", uint64_t{pool.GetUnbroadcastTxs().size()}); - return ret; -} - -static RPCHelpMan getmempoolinfo() -{ - return RPCHelpMan{"getmempoolinfo", - "\nReturns details on the active state of the TX memory pool.\n", - {}, - RPCResult{ - RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::BOOL, "loaded", "True if the mempool is fully loaded"}, - {RPCResult::Type::NUM, "size", "Current tx count"}, - {RPCResult::Type::NUM, "bytes", "Sum of all virtual transaction sizes as defined in BIP 141. Differs from actual serialized size because witness data is discounted"}, - {RPCResult::Type::NUM, "usage", "Total memory usage for the mempool"}, - {RPCResult::Type::STR_AMOUNT, "total_fee", "Total fees for the mempool in " + CURRENCY_UNIT + ", ignoring modified fees through prioritisetransaction"}, - {RPCResult::Type::NUM, "maxmempool", "Maximum memory usage for the mempool"}, - {RPCResult::Type::STR_AMOUNT, "mempoolminfee", "Minimum fee rate in " + CURRENCY_UNIT + "/kvB for tx to be accepted. Is the maximum of minrelaytxfee and minimum mempool fee"}, - {RPCResult::Type::STR_AMOUNT, "minrelaytxfee", "Current minimum relay fee for transactions"}, - {RPCResult::Type::NUM, "unbroadcastcount", "Current number of transactions that haven't passed initial broadcast yet"} - }}, - RPCExamples{ - HelpExampleCli("getmempoolinfo", "") - + HelpExampleRpc("getmempoolinfo", "") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - return MempoolInfoToJSON(EnsureAnyMemPool(request.context)); -}, - }; -} - static RPCHelpMan preciousblock() { return RPCHelpMan{"preciousblock", @@ -2034,9 +1613,9 @@ static RPCHelpMan getchaintxstats() } } - const CBlockIndex* pindexPast = pindex->GetAncestor(pindex->nHeight - blockcount); - int nTimeDiff = pindex->GetMedianTimePast() - pindexPast->GetMedianTimePast(); - int nTxDiff = pindex->nChainTx - pindexPast->nChainTx; + const CBlockIndex& past_block{*CHECK_NONFATAL(pindex->GetAncestor(pindex->nHeight - blockcount))}; + const int64_t nTimeDiff{pindex->GetMedianTimePast() - past_block.GetMedianTimePast()}; + const int nTxDiff = pindex->nChainTx - past_block.nChainTx; UniValue ret(UniValue::VOBJ); ret.pushKV("time", (int64_t)pindex->nTime); @@ -2177,8 +1756,7 @@ static RPCHelpMan getblockstats() { ChainstateManager& chainman = EnsureAnyChainman(request.context); LOCK(cs_main); - CBlockIndex* pindex{ParseHashOrHeight(request.params[0], chainman)}; - CHECK_NONFATAL(pindex != nullptr); + const CBlockIndex& pindex{*CHECK_NONFATAL(ParseHashOrHeight(request.params[0], chainman))}; std::set<std::string> stats; if (!request.params[1].isNull()) { @@ -2189,8 +1767,8 @@ static RPCHelpMan getblockstats() } } - const CBlock block = GetBlockChecked(pindex); - const CBlockUndo blockUndo = GetUndoChecked(pindex); + const CBlock& block = GetBlockChecked(chainman.m_blockman, &pindex); + const CBlockUndo& blockUndo = GetUndoChecked(chainman.m_blockman, &pindex); const bool do_all = stats.size() == 0; // Calculate everything if nothing selected (default) const bool do_mediantxsize = do_all || stats.count("mediantxsize") != 0; @@ -2308,25 +1886,25 @@ static RPCHelpMan getblockstats() ret_all.pushKV("avgfee", (block.vtx.size() > 1) ? totalfee / (block.vtx.size() - 1) : 0); ret_all.pushKV("avgfeerate", total_weight ? (totalfee * WITNESS_SCALE_FACTOR) / total_weight : 0); // Unit: sat/vbyte ret_all.pushKV("avgtxsize", (block.vtx.size() > 1) ? total_size / (block.vtx.size() - 1) : 0); - ret_all.pushKV("blockhash", pindex->GetBlockHash().GetHex()); + ret_all.pushKV("blockhash", pindex.GetBlockHash().GetHex()); ret_all.pushKV("feerate_percentiles", feerates_res); - ret_all.pushKV("height", (int64_t)pindex->nHeight); + ret_all.pushKV("height", (int64_t)pindex.nHeight); ret_all.pushKV("ins", inputs); ret_all.pushKV("maxfee", maxfee); ret_all.pushKV("maxfeerate", maxfeerate); ret_all.pushKV("maxtxsize", maxtxsize); ret_all.pushKV("medianfee", CalculateTruncatedMedian(fee_array)); - ret_all.pushKV("mediantime", pindex->GetMedianTimePast()); + ret_all.pushKV("mediantime", pindex.GetMedianTimePast()); ret_all.pushKV("mediantxsize", CalculateTruncatedMedian(txsize_array)); ret_all.pushKV("minfee", (minfee == MAX_MONEY) ? 0 : minfee); ret_all.pushKV("minfeerate", (minfeerate == MAX_MONEY) ? 0 : minfeerate); ret_all.pushKV("mintxsize", mintxsize == MAX_BLOCK_SERIALIZED_SIZE ? 0 : mintxsize); ret_all.pushKV("outs", outputs); - ret_all.pushKV("subsidy", GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())); + ret_all.pushKV("subsidy", GetBlockSubsidy(pindex.nHeight, Params().GetConsensus())); ret_all.pushKV("swtotal_size", swtotal_size); ret_all.pushKV("swtotal_weight", swtotal_weight); ret_all.pushKV("swtxs", swtxs); - ret_all.pushKV("time", pindex->GetBlockTime()); + ret_all.pushKV("time", pindex.GetBlockTime()); ret_all.pushKV("total_out", total_out); ret_all.pushKV("total_size", total_size); ret_all.pushKV("total_weight", total_weight); @@ -2352,41 +1930,6 @@ static RPCHelpMan getblockstats() }; } -static RPCHelpMan savemempool() -{ - return RPCHelpMan{"savemempool", - "\nDumps the mempool to disk. It will fail until the previous dump is fully loaded.\n", - {}, - RPCResult{ - RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR, "filename", "the directory and file where the mempool was saved"}, - }}, - RPCExamples{ - HelpExampleCli("savemempool", "") - + HelpExampleRpc("savemempool", "") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - const ArgsManager& args{EnsureAnyArgsman(request.context)}; - const CTxMemPool& mempool = EnsureAnyMemPool(request.context); - - if (!mempool.IsLoaded()) { - throw JSONRPCError(RPC_MISC_ERROR, "The mempool was not loaded yet"); - } - - if (!DumpMempool(mempool)) { - throw JSONRPCError(RPC_MISC_ERROR, "Unable to dump mempool to disk"); - } - - UniValue ret(UniValue::VOBJ); - ret.pushKV("filename", fs::path((args.GetDataDirNet() / "mempool.dat")).u8string()); - - return ret; -}, - }; -} - namespace { //! Search for a given set of pubkey scripts bool FindScriptPubKey(std::atomic<int>& scan_progress, const std::atomic<bool>& should_abort, int64_t& count, CCoinsViewCursor* cursor, const std::set<CScript>& needles, std::map<COutPoint, Coin>& out_results, std::function<void()>& interruption_point) @@ -2572,17 +2115,15 @@ static RPCHelpMan scantxoutset() g_should_abort_scan = false; int64_t count = 0; std::unique_ptr<CCoinsViewCursor> pcursor; - CBlockIndex* tip; + const CBlockIndex* tip; NodeContext& node = EnsureAnyNodeContext(request.context); { ChainstateManager& chainman = EnsureChainman(node); LOCK(cs_main); CChainState& active_chainstate = chainman.ActiveChainstate(); active_chainstate.ForceFlushStateToDisk(); - pcursor = active_chainstate.CoinsDB().Cursor(); - CHECK_NONFATAL(pcursor); - tip = active_chainstate.m_chain.Tip(); - CHECK_NONFATAL(tip); + pcursor = CHECK_NONFATAL(active_chainstate.CoinsDB().Cursor()); + tip = CHECK_NONFATAL(active_chainstate.m_chain.Tip()); } bool res = FindScriptPubKey(g_scan_progress, g_should_abort_scan, count, pcursor.get(), needles, coins, node.rpc_interruption_point); result.pushKV("success", res); @@ -2740,6 +2281,12 @@ static RPCHelpMan dumptxoutset() FILE* file{fsbridge::fopen(temppath, "wb")}; CAutoFile afile{file, SER_DISK, CLIENT_VERSION}; + if (afile.IsNull()) { + throw JSONRPCError( + RPC_INVALID_PARAMETER, + "Couldn't open file " + temppath.u8string() + " for writing."); + } + NodeContext& node = EnsureAnyNodeContext(request.context); UniValue result = CreateUTXOSnapshot( node, node.chainman->ActiveChainstate(), afile, path, temppath); @@ -2760,7 +2307,7 @@ UniValue CreateUTXOSnapshot( { std::unique_ptr<CCoinsViewCursor> pcursor; CCoinsStats stats{CoinStatsHashType::HASH_SERIALIZED}; - CBlockIndex* tip; + const CBlockIndex* tip; { // We need to lock cs_main to ensure that the coinsdb isn't written to @@ -2784,8 +2331,7 @@ UniValue CreateUTXOSnapshot( } pcursor = chainstate.CoinsDB().Cursor(); - tip = chainstate.m_blockman.LookupBlockIndex(stats.hashBlock); - CHECK_NONFATAL(tip); + tip = CHECK_NONFATAL(chainstate.m_blockman.LookupBlockIndex(stats.hashBlock)); } LOG_TIME_SECONDS(strprintf("writing UTXO snapshot at height %s (%s) to file %s (via %s)", @@ -2825,49 +2371,36 @@ UniValue CreateUTXOSnapshot( return result; } -void RegisterBlockchainRPCCommands(CRPCTable &t) -{ -// clang-format off -static const CRPCCommand commands[] = -{ // category actor (function) - // --------------------- ------------------------ - { "blockchain", &getblockchaininfo, }, - { "blockchain", &getchaintxstats, }, - { "blockchain", &getblockstats, }, - { "blockchain", &getbestblockhash, }, - { "blockchain", &getblockcount, }, - { "blockchain", &getblock, }, - { "blockchain", &getblockfrompeer, }, - { "blockchain", &getblockhash, }, - { "blockchain", &getblockheader, }, - { "blockchain", &getchaintips, }, - { "blockchain", &getdifficulty, }, - { "blockchain", &getdeploymentinfo, }, - { "blockchain", &getmempoolancestors, }, - { "blockchain", &getmempooldescendants, }, - { "blockchain", &getmempoolentry, }, - { "blockchain", &getmempoolinfo, }, - { "blockchain", &getrawmempool, }, - { "blockchain", &gettxout, }, - { "blockchain", &gettxoutsetinfo, }, - { "blockchain", &pruneblockchain, }, - { "blockchain", &savemempool, }, - { "blockchain", &verifychain, }, - - { "blockchain", &preciousblock, }, - { "blockchain", &scantxoutset, }, - { "blockchain", &getblockfilter, }, - - /* Not shown in help */ - { "hidden", &invalidateblock, }, - { "hidden", &reconsiderblock, }, - { "hidden", &waitfornewblock, }, - { "hidden", &waitforblock, }, - { "hidden", &waitforblockheight, }, - { "hidden", &syncwithvalidationinterfacequeue, }, - { "hidden", &dumptxoutset, }, -}; -// clang-format on +void RegisterBlockchainRPCCommands(CRPCTable& t) +{ + static const CRPCCommand commands[]{ + {"blockchain", &getblockchaininfo}, + {"blockchain", &getchaintxstats}, + {"blockchain", &getblockstats}, + {"blockchain", &getbestblockhash}, + {"blockchain", &getblockcount}, + {"blockchain", &getblock}, + {"blockchain", &getblockfrompeer}, + {"blockchain", &getblockhash}, + {"blockchain", &getblockheader}, + {"blockchain", &getchaintips}, + {"blockchain", &getdifficulty}, + {"blockchain", &getdeploymentinfo}, + {"blockchain", &gettxout}, + {"blockchain", &gettxoutsetinfo}, + {"blockchain", &pruneblockchain}, + {"blockchain", &verifychain}, + {"blockchain", &preciousblock}, + {"blockchain", &scantxoutset}, + {"blockchain", &getblockfilter}, + {"hidden", &invalidateblock}, + {"hidden", &reconsiderblock}, + {"hidden", &waitfornewblock}, + {"hidden", &waitforblock}, + {"hidden", &waitforblockheight}, + {"hidden", &syncwithvalidationinterfacequeue}, + {"hidden", &dumptxoutset}, + }; for (const auto& c : commands) { t.appendCommand(c.name, &c); } diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index 1f51d7c1ad..5fbd9d5fd3 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -10,6 +10,7 @@ #include <fs.h> #include <streams.h> #include <sync.h> +#include <validation.h> #include <any> #include <stdint.h> @@ -20,7 +21,6 @@ extern RecursiveMutex cs_main; class CBlock; class CBlockIndex; class CChainState; -class CTxMemPool; class UniValue; namespace node { struct NodeContext; @@ -40,13 +40,7 @@ double GetDifficulty(const CBlockIndex* blockindex); void RPCNotifyBlockChange(const CBlockIndex*); /** Block description to JSON */ -UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, TxVerbosity verbosity) LOCKS_EXCLUDED(cs_main); - -/** Mempool information to JSON */ -UniValue MempoolInfoToJSON(const CTxMemPool& pool); - -/** Mempool to JSON */ -UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose = false, bool include_mempool_sequence = false); +UniValue blockToJSON(node::BlockManager& blockman, const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, TxVerbosity verbosity) LOCKS_EXCLUDED(cs_main); /** Block header to JSON */ UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex) LOCKS_EXCLUDED(cs_main); diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index c480a093a4..23e9d4074c 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -142,6 +142,10 @@ static const CRPCConvertParam vRPCConvertParams[] = { "send", 1, "conf_target" }, { "send", 3, "fee_rate"}, { "send", 4, "options" }, + { "sendall", 0, "recipients" }, + { "sendall", 1, "conf_target" }, + { "sendall", 3, "fee_rate"}, + { "sendall", 4, "options" }, { "importprivkey", 2, "rescan" }, { "importaddress", 2, "rescan" }, { "importaddress", 3, "p2sh" }, diff --git a/src/rpc/external_signer.cpp b/src/rpc/external_signer.cpp index 60ec15e904..4de7fc4205 100644 --- a/src/rpc/external_signer.cpp +++ b/src/rpc/external_signer.cpp @@ -22,7 +22,7 @@ static RPCHelpMan enumeratesigners() RPCResult{ RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::ARR, "signers", /* optional */ false, "", + {RPCResult::Type::ARR, "signers", /*optional=*/false, "", { {RPCResult::Type::OBJ, "", "", { @@ -62,15 +62,11 @@ static RPCHelpMan enumeratesigners() }; } -void RegisterSignerRPCCommands(CRPCTable &t) +void RegisterSignerRPCCommands(CRPCTable& t) { -// clang-format off -static const CRPCCommand commands[] = -{ // category actor (function) - // --------------------- ------------------------ - { "signer", &enumeratesigners, }, -}; -// clang-format on + static const CRPCCommand commands[]{ + {"signer", &enumeratesigners}, + }; for (const auto& c : commands) { t.appendCommand(c.name, &c); } diff --git a/src/rpc/fees.cpp b/src/rpc/fees.cpp new file mode 100644 index 0000000000..bfec0780aa --- /dev/null +++ b/src/rpc/fees.cpp @@ -0,0 +1,236 @@ +// Copyright (c) 2010 Satoshi Nakamoto +// Copyright (c) 2009-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <core_io.h> +#include <policy/feerate.h> +#include <policy/fees.h> +#include <policy/policy.h> +#include <rpc/protocol.h> +#include <rpc/request.h> +#include <rpc/server.h> +#include <rpc/server_util.h> +#include <rpc/util.h> +#include <txmempool.h> +#include <univalue.h> +#include <util/fees.h> +#include <util/system.h> +#include <validation.h> + +#include <algorithm> +#include <array> +#include <cmath> +#include <string> + +namespace node { +struct NodeContext; +} + +using node::NodeContext; + +static RPCHelpMan estimatesmartfee() +{ + return RPCHelpMan{"estimatesmartfee", + "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" + "confirmation within conf_target blocks if possible and return the number of blocks\n" + "for which the estimate is valid. Uses virtual transaction size as defined\n" + "in BIP 141 (witness data is discounted).\n", + { + {"conf_target", RPCArg::Type::NUM, RPCArg::Optional::NO, "Confirmation target in blocks (1 - 1008)"}, + {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"conservative"}, "The fee estimate mode.\n" + "Whether to return a more conservative estimate which also satisfies\n" + "a longer history. A conservative estimate potentially returns a\n" + "higher feerate and is more likely to be sufficient for the desired\n" + "target, but is not as responsive to short term drops in the\n" + "prevailing fee market. Must be one of (case insensitive):\n" + "\"" + FeeModes("\"\n\"") + "\""}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::NUM, "feerate", /*optional=*/true, "estimate fee rate in " + CURRENCY_UNIT + "/kvB (only present if no errors were encountered)"}, + {RPCResult::Type::ARR, "errors", /*optional=*/true, "Errors encountered during processing (if there are any)", + { + {RPCResult::Type::STR, "", "error"}, + }}, + {RPCResult::Type::NUM, "blocks", "block number where estimate was found\n" + "The request target will be clamped between 2 and the highest target\n" + "fee estimation is able to return based on how long it has been running.\n" + "An error is returned if not enough transactions and blocks\n" + "have been observed to make an estimate for any number of blocks."}, + }}, + RPCExamples{ + HelpExampleCli("estimatesmartfee", "6") + + HelpExampleRpc("estimatesmartfee", "6") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + RPCTypeCheck(request.params, {UniValue::VNUM, UniValue::VSTR}); + RPCTypeCheckArgument(request.params[0], UniValue::VNUM); + + CBlockPolicyEstimator& fee_estimator = EnsureAnyFeeEstimator(request.context); + const NodeContext& node = EnsureAnyNodeContext(request.context); + const CTxMemPool& mempool = EnsureMemPool(node); + + unsigned int max_target = fee_estimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); + unsigned int conf_target = ParseConfirmTarget(request.params[0], max_target); + bool conservative = true; + if (!request.params[1].isNull()) { + FeeEstimateMode fee_mode; + if (!FeeModeFromString(request.params[1].get_str(), fee_mode)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, InvalidEstimateModeErrorMessage()); + } + if (fee_mode == FeeEstimateMode::ECONOMICAL) conservative = false; + } + + UniValue result(UniValue::VOBJ); + UniValue errors(UniValue::VARR); + FeeCalculation feeCalc; + CFeeRate feeRate{fee_estimator.estimateSmartFee(conf_target, &feeCalc, conservative)}; + if (feeRate != CFeeRate(0)) { + CFeeRate min_mempool_feerate{mempool.GetMinFee(gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000)}; + CFeeRate min_relay_feerate{::minRelayTxFee}; + feeRate = std::max({feeRate, min_mempool_feerate, min_relay_feerate}); + result.pushKV("feerate", ValueFromAmount(feeRate.GetFeePerK())); + } else { + errors.push_back("Insufficient data or no feerate found"); + result.pushKV("errors", errors); + } + result.pushKV("blocks", feeCalc.returnedTarget); + return result; + }, + }; +} + +static RPCHelpMan estimaterawfee() +{ + return RPCHelpMan{"estimaterawfee", + "\nWARNING: This interface is unstable and may disappear or change!\n" + "\nWARNING: This is an advanced API call that is tightly coupled to the specific\n" + "implementation of fee estimation. The parameters it can be called with\n" + "and the results it returns will change if the internal implementation changes.\n" + "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" + "confirmation within conf_target blocks if possible. Uses virtual transaction size as\n" + "defined in BIP 141 (witness data is discounted).\n", + { + {"conf_target", RPCArg::Type::NUM, RPCArg::Optional::NO, "Confirmation target in blocks (1 - 1008)"}, + {"threshold", RPCArg::Type::NUM, RPCArg::Default{0.95}, "The proportion of transactions in a given feerate range that must have been\n" + "confirmed within conf_target in order to consider those feerates as high enough and proceed to check\n" + "lower buckets."}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "Results are returned for any horizon which tracks blocks up to the confirmation target", + { + {RPCResult::Type::OBJ, "short", /*optional=*/true, "estimate for short time horizon", + { + {RPCResult::Type::NUM, "feerate", /*optional=*/true, "estimate fee rate in " + CURRENCY_UNIT + "/kvB"}, + {RPCResult::Type::NUM, "decay", "exponential decay (per block) for historical moving average of confirmation data"}, + {RPCResult::Type::NUM, "scale", "The resolution of confirmation targets at this time horizon"}, + {RPCResult::Type::OBJ, "pass", /*optional=*/true, "information about the lowest range of feerates to succeed in meeting the threshold", + { + {RPCResult::Type::NUM, "startrange", "start of feerate range"}, + {RPCResult::Type::NUM, "endrange", "end of feerate range"}, + {RPCResult::Type::NUM, "withintarget", "number of txs over history horizon in the feerate range that were confirmed within target"}, + {RPCResult::Type::NUM, "totalconfirmed", "number of txs over history horizon in the feerate range that were confirmed at any point"}, + {RPCResult::Type::NUM, "inmempool", "current number of txs in mempool in the feerate range unconfirmed for at least target blocks"}, + {RPCResult::Type::NUM, "leftmempool", "number of txs over history horizon in the feerate range that left mempool unconfirmed after target"}, + }}, + {RPCResult::Type::OBJ, "fail", /*optional=*/true, "information about the highest range of feerates to fail to meet the threshold", + { + {RPCResult::Type::ELISION, "", ""}, + }}, + {RPCResult::Type::ARR, "errors", /*optional=*/true, "Errors encountered during processing (if there are any)", + { + {RPCResult::Type::STR, "error", ""}, + }}, + }}, + {RPCResult::Type::OBJ, "medium", /*optional=*/true, "estimate for medium time horizon", + { + {RPCResult::Type::ELISION, "", ""}, + }}, + {RPCResult::Type::OBJ, "long", /*optional=*/true, "estimate for long time horizon", + { + {RPCResult::Type::ELISION, "", ""}, + }}, + }}, + RPCExamples{ + HelpExampleCli("estimaterawfee", "6 0.9") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + RPCTypeCheck(request.params, {UniValue::VNUM, UniValue::VNUM}, true); + RPCTypeCheckArgument(request.params[0], UniValue::VNUM); + + CBlockPolicyEstimator& fee_estimator = EnsureAnyFeeEstimator(request.context); + + unsigned int max_target = fee_estimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); + unsigned int conf_target = ParseConfirmTarget(request.params[0], max_target); + double threshold = 0.95; + if (!request.params[1].isNull()) { + threshold = request.params[1].get_real(); + } + if (threshold < 0 || threshold > 1) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid threshold"); + } + + UniValue result(UniValue::VOBJ); + + for (const FeeEstimateHorizon horizon : ALL_FEE_ESTIMATE_HORIZONS) { + CFeeRate feeRate; + EstimationResult buckets; + + // Only output results for horizons which track the target + if (conf_target > fee_estimator.HighestTargetTracked(horizon)) continue; + + feeRate = fee_estimator.estimateRawFee(conf_target, threshold, horizon, &buckets); + UniValue horizon_result(UniValue::VOBJ); + UniValue errors(UniValue::VARR); + UniValue passbucket(UniValue::VOBJ); + passbucket.pushKV("startrange", round(buckets.pass.start)); + passbucket.pushKV("endrange", round(buckets.pass.end)); + passbucket.pushKV("withintarget", round(buckets.pass.withinTarget * 100.0) / 100.0); + passbucket.pushKV("totalconfirmed", round(buckets.pass.totalConfirmed * 100.0) / 100.0); + passbucket.pushKV("inmempool", round(buckets.pass.inMempool * 100.0) / 100.0); + passbucket.pushKV("leftmempool", round(buckets.pass.leftMempool * 100.0) / 100.0); + UniValue failbucket(UniValue::VOBJ); + failbucket.pushKV("startrange", round(buckets.fail.start)); + failbucket.pushKV("endrange", round(buckets.fail.end)); + failbucket.pushKV("withintarget", round(buckets.fail.withinTarget * 100.0) / 100.0); + failbucket.pushKV("totalconfirmed", round(buckets.fail.totalConfirmed * 100.0) / 100.0); + failbucket.pushKV("inmempool", round(buckets.fail.inMempool * 100.0) / 100.0); + failbucket.pushKV("leftmempool", round(buckets.fail.leftMempool * 100.0) / 100.0); + + // CFeeRate(0) is used to indicate error as a return value from estimateRawFee + if (feeRate != CFeeRate(0)) { + horizon_result.pushKV("feerate", ValueFromAmount(feeRate.GetFeePerK())); + horizon_result.pushKV("decay", buckets.decay); + horizon_result.pushKV("scale", (int)buckets.scale); + horizon_result.pushKV("pass", passbucket); + // buckets.fail.start == -1 indicates that all buckets passed, there is no fail bucket to output + if (buckets.fail.start != -1) horizon_result.pushKV("fail", failbucket); + } else { + // Output only information that is still meaningful in the event of error + horizon_result.pushKV("decay", buckets.decay); + horizon_result.pushKV("scale", (int)buckets.scale); + horizon_result.pushKV("fail", failbucket); + errors.push_back("Insufficient data or no feerate found which meets threshold"); + horizon_result.pushKV("errors", errors); + } + result.pushKV(StringForFeeEstimateHorizon(horizon), horizon_result); + } + return result; + }, + }; +} + +void RegisterFeeRPCCommands(CRPCTable& t) +{ + static const CRPCCommand commands[]{ + {"util", &estimatesmartfee}, + {"hidden", &estimaterawfee}, + }; + for (const auto& c : commands) { + t.appendCommand(c.name, &c); + } +} diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp new file mode 100644 index 0000000000..27080d3881 --- /dev/null +++ b/src/rpc/mempool.cpp @@ -0,0 +1,687 @@ +// Copyright (c) 2010 Satoshi Nakamoto +// Copyright (c) 2009-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <rpc/blockchain.h> + +#include <core_io.h> +#include <fs.h> +#include <policy/rbf.h> +#include <primitives/transaction.h> +#include <rpc/server.h> +#include <rpc/server_util.h> +#include <rpc/util.h> +#include <txmempool.h> +#include <univalue.h> +#include <util/moneystr.h> +#include <validation.h> + +using node::DEFAULT_MAX_RAW_TX_FEE_RATE; +using node::NodeContext; + +static RPCHelpMan sendrawtransaction() +{ + return RPCHelpMan{"sendrawtransaction", + "\nSubmit a raw transaction (serialized, hex-encoded) to local node and network.\n" + "\nThe transaction will be sent unconditionally to all peers, so using sendrawtransaction\n" + "for manual rebroadcast may degrade privacy by leaking the transaction's origin, as\n" + "nodes will normally not rebroadcast non-wallet transactions already in their mempool.\n" + "\nA specific exception, RPC_TRANSACTION_ALREADY_IN_CHAIN, may throw if the transaction cannot be added to the mempool.\n" + "\nRelated RPCs: createrawtransaction, signrawtransactionwithkey\n", + { + {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex string of the raw transaction"}, + {"maxfeerate", RPCArg::Type::AMOUNT, RPCArg::Default{FormatMoney(DEFAULT_MAX_RAW_TX_FEE_RATE.GetFeePerK())}, + "Reject transactions whose fee rate is higher than the specified value, expressed in " + CURRENCY_UNIT + + "/kvB.\nSet to 0 to accept any fee rate.\n"}, + }, + RPCResult{ + RPCResult::Type::STR_HEX, "", "The transaction hash in hex" + }, + RPCExamples{ + "\nCreate a transaction\n" + + HelpExampleCli("createrawtransaction", "\"[{\\\"txid\\\" : \\\"mytxid\\\",\\\"vout\\\":0}]\" \"{\\\"myaddress\\\":0.01}\"") + + "Sign the transaction, and get back the hex\n" + + HelpExampleCli("signrawtransactionwithwallet", "\"myhex\"") + + "\nSend the transaction (signed hex)\n" + + HelpExampleCli("sendrawtransaction", "\"signedhex\"") + + "\nAs a JSON-RPC call\n" + + HelpExampleRpc("sendrawtransaction", "\"signedhex\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + RPCTypeCheck(request.params, { + UniValue::VSTR, + UniValueType(), // VNUM or VSTR, checked inside AmountFromValue() + }); + + CMutableTransaction mtx; + if (!DecodeHexTx(mtx, request.params[0].get_str())) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed. Make sure the tx has at least one input."); + } + CTransactionRef tx(MakeTransactionRef(std::move(mtx))); + + const CFeeRate max_raw_tx_fee_rate = request.params[1].isNull() ? + DEFAULT_MAX_RAW_TX_FEE_RATE : + CFeeRate(AmountFromValue(request.params[1])); + + int64_t virtual_size = GetVirtualTransactionSize(*tx); + CAmount max_raw_tx_fee = max_raw_tx_fee_rate.GetFee(virtual_size); + + std::string err_string; + AssertLockNotHeld(cs_main); + NodeContext& node = EnsureAnyNodeContext(request.context); + const TransactionError err = BroadcastTransaction(node, tx, err_string, max_raw_tx_fee, /*relay=*/true, /*wait_callback=*/true); + if (TransactionError::OK != err) { + throw JSONRPCTransactionError(err, err_string); + } + + return tx->GetHash().GetHex(); + }, + }; +} + +static RPCHelpMan testmempoolaccept() +{ + return RPCHelpMan{"testmempoolaccept", + "\nReturns result of mempool acceptance tests indicating if raw transaction(s) (serialized, hex-encoded) would be accepted by mempool.\n" + "\nIf multiple transactions are passed in, parents must come before children and package policies apply: the transactions cannot conflict with any mempool transactions or each other.\n" + "\nIf one transaction fails, other transactions may not be fully validated (the 'allowed' key will be blank).\n" + "\nThe maximum number of transactions allowed is " + ToString(MAX_PACKAGE_COUNT) + ".\n" + "\nThis checks if transactions violate the consensus or policy rules.\n" + "\nSee sendrawtransaction call.\n", + { + {"rawtxs", RPCArg::Type::ARR, RPCArg::Optional::NO, "An array of hex strings of raw transactions.", + { + {"rawtx", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, + }, + }, + {"maxfeerate", RPCArg::Type::AMOUNT, RPCArg::Default{FormatMoney(DEFAULT_MAX_RAW_TX_FEE_RATE.GetFeePerK())}, + "Reject transactions whose fee rate is higher than the specified value, expressed in " + CURRENCY_UNIT + "/kvB\n"}, + }, + RPCResult{ + RPCResult::Type::ARR, "", "The result of the mempool acceptance test for each raw transaction in the input array.\n" + "Returns results for each transaction in the same order they were passed in.\n" + "Transactions that cannot be fully validated due to failures in other transactions will not contain an 'allowed' result.\n", + { + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR_HEX, "txid", "The transaction hash in hex"}, + {RPCResult::Type::STR_HEX, "wtxid", "The transaction witness hash in hex"}, + {RPCResult::Type::STR, "package-error", /*optional=*/true, "Package validation error, if any (only possible if rawtxs had more than 1 transaction)."}, + {RPCResult::Type::BOOL, "allowed", /*optional=*/true, "Whether this tx would be accepted to the mempool and pass client-specified maxfeerate. " + "If not present, the tx was not fully validated due to a failure in another tx in the list."}, + {RPCResult::Type::NUM, "vsize", /*optional=*/true, "Virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted (only present when 'allowed' is true)"}, + {RPCResult::Type::OBJ, "fees", /*optional=*/true, "Transaction fees (only present if 'allowed' is true)", + { + {RPCResult::Type::STR_AMOUNT, "base", "transaction fee in " + CURRENCY_UNIT}, + }}, + {RPCResult::Type::STR, "reject-reason", /*optional=*/true, "Rejection string (only present when 'allowed' is false)"}, + }}, + } + }, + RPCExamples{ + "\nCreate a transaction\n" + + HelpExampleCli("createrawtransaction", "\"[{\\\"txid\\\" : \\\"mytxid\\\",\\\"vout\\\":0}]\" \"{\\\"myaddress\\\":0.01}\"") + + "Sign the transaction, and get back the hex\n" + + HelpExampleCli("signrawtransactionwithwallet", "\"myhex\"") + + "\nTest acceptance of the transaction (signed hex)\n" + + HelpExampleCli("testmempoolaccept", R"('["signedhex"]')") + + "\nAs a JSON-RPC call\n" + + HelpExampleRpc("testmempoolaccept", "[\"signedhex\"]") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + RPCTypeCheck(request.params, { + UniValue::VARR, + UniValueType(), // VNUM or VSTR, checked inside AmountFromValue() + }); + const UniValue raw_transactions = request.params[0].get_array(); + if (raw_transactions.size() < 1 || raw_transactions.size() > MAX_PACKAGE_COUNT) { + throw JSONRPCError(RPC_INVALID_PARAMETER, + "Array must contain between 1 and " + ToString(MAX_PACKAGE_COUNT) + " transactions."); + } + + const CFeeRate max_raw_tx_fee_rate = request.params[1].isNull() ? + DEFAULT_MAX_RAW_TX_FEE_RATE : + CFeeRate(AmountFromValue(request.params[1])); + + std::vector<CTransactionRef> txns; + txns.reserve(raw_transactions.size()); + for (const auto& rawtx : raw_transactions.getValues()) { + CMutableTransaction mtx; + if (!DecodeHexTx(mtx, rawtx.get_str())) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, + "TX decode failed: " + rawtx.get_str() + " Make sure the tx has at least one input."); + } + txns.emplace_back(MakeTransactionRef(std::move(mtx))); + } + + NodeContext& node = EnsureAnyNodeContext(request.context); + CTxMemPool& mempool = EnsureMemPool(node); + ChainstateManager& chainman = EnsureChainman(node); + CChainState& chainstate = chainman.ActiveChainstate(); + const PackageMempoolAcceptResult package_result = [&] { + LOCK(::cs_main); + if (txns.size() > 1) return ProcessNewPackage(chainstate, mempool, txns, /*test_accept=*/true); + return PackageMempoolAcceptResult(txns[0]->GetWitnessHash(), + chainman.ProcessTransaction(txns[0], /*test_accept=*/true)); + }(); + + UniValue rpc_result(UniValue::VARR); + // We will check transaction fees while we iterate through txns in order. If any transaction fee + // exceeds maxfeerate, we will leave the rest of the validation results blank, because it + // doesn't make sense to return a validation result for a transaction if its ancestor(s) would + // not be submitted. + bool exit_early{false}; + for (const auto& tx : txns) { + UniValue result_inner(UniValue::VOBJ); + result_inner.pushKV("txid", tx->GetHash().GetHex()); + result_inner.pushKV("wtxid", tx->GetWitnessHash().GetHex()); + if (package_result.m_state.GetResult() == PackageValidationResult::PCKG_POLICY) { + result_inner.pushKV("package-error", package_result.m_state.GetRejectReason()); + } + auto it = package_result.m_tx_results.find(tx->GetWitnessHash()); + if (exit_early || it == package_result.m_tx_results.end()) { + // Validation unfinished. Just return the txid and wtxid. + rpc_result.push_back(result_inner); + continue; + } + const auto& tx_result = it->second; + // Package testmempoolaccept doesn't allow transactions to already be in the mempool. + CHECK_NONFATAL(tx_result.m_result_type != MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + if (tx_result.m_result_type == MempoolAcceptResult::ResultType::VALID) { + const CAmount fee = tx_result.m_base_fees.value(); + // Check that fee does not exceed maximum fee + const int64_t virtual_size = tx_result.m_vsize.value(); + const CAmount max_raw_tx_fee = max_raw_tx_fee_rate.GetFee(virtual_size); + if (max_raw_tx_fee && fee > max_raw_tx_fee) { + result_inner.pushKV("allowed", false); + result_inner.pushKV("reject-reason", "max-fee-exceeded"); + exit_early = true; + } else { + // Only return the fee and vsize if the transaction would pass ATMP. + // These can be used to calculate the feerate. + result_inner.pushKV("allowed", true); + result_inner.pushKV("vsize", virtual_size); + UniValue fees(UniValue::VOBJ); + fees.pushKV("base", ValueFromAmount(fee)); + result_inner.pushKV("fees", fees); + } + } else { + result_inner.pushKV("allowed", false); + const TxValidationState state = tx_result.m_state; + if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { + result_inner.pushKV("reject-reason", "missing-inputs"); + } else { + result_inner.pushKV("reject-reason", state.GetRejectReason()); + } + } + rpc_result.push_back(result_inner); + } + return rpc_result; + }, + }; +} + +static std::vector<RPCResult> MempoolEntryDescription() +{ + return { + RPCResult{RPCResult::Type::NUM, "vsize", "virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted."}, + RPCResult{RPCResult::Type::NUM, "weight", "transaction weight as defined in BIP 141."}, + RPCResult{RPCResult::Type::STR_AMOUNT, "fee", /*optional=*/true, + "transaction fee, denominated in " + CURRENCY_UNIT + " (DEPRECATED, returned only if config option -deprecatedrpc=fees is passed)"}, + RPCResult{RPCResult::Type::STR_AMOUNT, "modifiedfee", /*optional=*/true, + "transaction fee with fee deltas used for mining priority, denominated in " + CURRENCY_UNIT + + " (DEPRECATED, returned only if config option -deprecatedrpc=fees is passed)"}, + RPCResult{RPCResult::Type::NUM_TIME, "time", "local time transaction entered pool in seconds since 1 Jan 1970 GMT"}, + RPCResult{RPCResult::Type::NUM, "height", "block height when transaction entered pool"}, + RPCResult{RPCResult::Type::NUM, "descendantcount", "number of in-mempool descendant transactions (including this one)"}, + RPCResult{RPCResult::Type::NUM, "descendantsize", "virtual transaction size of in-mempool descendants (including this one)"}, + RPCResult{RPCResult::Type::STR_AMOUNT, "descendantfees", /*optional=*/true, + "transaction fees of in-mempool descendants (including this one) with fee deltas used for mining priority, denominated in " + + CURRENCY_ATOM + "s (DEPRECATED, returned only if config option -deprecatedrpc=fees is passed)"}, + RPCResult{RPCResult::Type::NUM, "ancestorcount", "number of in-mempool ancestor transactions (including this one)"}, + RPCResult{RPCResult::Type::NUM, "ancestorsize", "virtual transaction size of in-mempool ancestors (including this one)"}, + RPCResult{RPCResult::Type::STR_AMOUNT, "ancestorfees", /*optional=*/true, + "transaction fees of in-mempool ancestors (including this one) with fee deltas used for mining priority, denominated in " + + CURRENCY_ATOM + "s (DEPRECATED, returned only if config option -deprecatedrpc=fees is passed)"}, + RPCResult{RPCResult::Type::STR_HEX, "wtxid", "hash of serialized transaction, including witness data"}, + RPCResult{RPCResult::Type::OBJ, "fees", "", + { + RPCResult{RPCResult::Type::STR_AMOUNT, "base", "transaction fee, denominated in " + CURRENCY_UNIT}, + RPCResult{RPCResult::Type::STR_AMOUNT, "modified", "transaction fee with fee deltas used for mining priority, denominated in " + CURRENCY_UNIT}, + RPCResult{RPCResult::Type::STR_AMOUNT, "ancestor", "transaction fees of in-mempool ancestors (including this one) with fee deltas used for mining priority, denominated in " + CURRENCY_UNIT}, + RPCResult{RPCResult::Type::STR_AMOUNT, "descendant", "transaction fees of in-mempool descendants (including this one) with fee deltas used for mining priority, denominated in " + CURRENCY_UNIT}, + }}, + RPCResult{RPCResult::Type::ARR, "depends", "unconfirmed transactions used as inputs for this transaction", + {RPCResult{RPCResult::Type::STR_HEX, "transactionid", "parent transaction id"}}}, + RPCResult{RPCResult::Type::ARR, "spentby", "unconfirmed transactions spending outputs from this transaction", + {RPCResult{RPCResult::Type::STR_HEX, "transactionid", "child transaction id"}}}, + RPCResult{RPCResult::Type::BOOL, "bip125-replaceable", "Whether this transaction could be replaced due to BIP125 (replace-by-fee)"}, + RPCResult{RPCResult::Type::BOOL, "unbroadcast", "Whether this transaction is currently unbroadcast (initial broadcast not yet acknowledged by any peers)"}, + }; +} + +static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPoolEntry& e) EXCLUSIVE_LOCKS_REQUIRED(pool.cs) +{ + AssertLockHeld(pool.cs); + + info.pushKV("vsize", (int)e.GetTxSize()); + info.pushKV("weight", (int)e.GetTxWeight()); + // TODO: top-level fee fields are deprecated. deprecated_fee_fields_enabled blocks should be removed in v24 + const bool deprecated_fee_fields_enabled{IsDeprecatedRPCEnabled("fees")}; + if (deprecated_fee_fields_enabled) { + info.pushKV("fee", ValueFromAmount(e.GetFee())); + info.pushKV("modifiedfee", ValueFromAmount(e.GetModifiedFee())); + } + info.pushKV("time", count_seconds(e.GetTime())); + info.pushKV("height", (int)e.GetHeight()); + info.pushKV("descendantcount", e.GetCountWithDescendants()); + info.pushKV("descendantsize", e.GetSizeWithDescendants()); + if (deprecated_fee_fields_enabled) { + info.pushKV("descendantfees", e.GetModFeesWithDescendants()); + } + info.pushKV("ancestorcount", e.GetCountWithAncestors()); + info.pushKV("ancestorsize", e.GetSizeWithAncestors()); + if (deprecated_fee_fields_enabled) { + info.pushKV("ancestorfees", e.GetModFeesWithAncestors()); + } + info.pushKV("wtxid", pool.vTxHashes[e.vTxHashesIdx].first.ToString()); + + UniValue fees(UniValue::VOBJ); + fees.pushKV("base", ValueFromAmount(e.GetFee())); + fees.pushKV("modified", ValueFromAmount(e.GetModifiedFee())); + fees.pushKV("ancestor", ValueFromAmount(e.GetModFeesWithAncestors())); + fees.pushKV("descendant", ValueFromAmount(e.GetModFeesWithDescendants())); + info.pushKV("fees", fees); + + const CTransaction& tx = e.GetTx(); + std::set<std::string> setDepends; + for (const CTxIn& txin : tx.vin) + { + if (pool.exists(GenTxid::Txid(txin.prevout.hash))) + setDepends.insert(txin.prevout.hash.ToString()); + } + + UniValue depends(UniValue::VARR); + for (const std::string& dep : setDepends) + { + depends.push_back(dep); + } + + info.pushKV("depends", depends); + + UniValue spent(UniValue::VARR); + const CTxMemPool::txiter& it = pool.mapTx.find(tx.GetHash()); + const CTxMemPoolEntry::Children& children = it->GetMemPoolChildrenConst(); + for (const CTxMemPoolEntry& child : children) { + spent.push_back(child.GetTx().GetHash().ToString()); + } + + info.pushKV("spentby", spent); + + // Add opt-in RBF status + bool rbfStatus = false; + RBFTransactionState rbfState = IsRBFOptIn(tx, pool); + if (rbfState == RBFTransactionState::UNKNOWN) { + throw JSONRPCError(RPC_MISC_ERROR, "Transaction is not in mempool"); + } else if (rbfState == RBFTransactionState::REPLACEABLE_BIP125) { + rbfStatus = true; + } + + info.pushKV("bip125-replaceable", rbfStatus); + info.pushKV("unbroadcast", pool.IsUnbroadcastTx(tx.GetHash())); +} + +UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose, bool include_mempool_sequence) +{ + if (verbose) { + if (include_mempool_sequence) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Verbose results cannot contain mempool sequence values."); + } + LOCK(pool.cs); + UniValue o(UniValue::VOBJ); + for (const CTxMemPoolEntry& e : pool.mapTx) { + const uint256& hash = e.GetTx().GetHash(); + UniValue info(UniValue::VOBJ); + entryToJSON(pool, info, e); + // Mempool has unique entries so there is no advantage in using + // UniValue::pushKV, which checks if the key already exists in O(N). + // UniValue::__pushKV is used instead which currently is O(1). + o.__pushKV(hash.ToString(), info); + } + return o; + } else { + uint64_t mempool_sequence; + std::vector<uint256> vtxid; + { + LOCK(pool.cs); + pool.queryHashes(vtxid); + mempool_sequence = pool.GetSequence(); + } + UniValue a(UniValue::VARR); + for (const uint256& hash : vtxid) + a.push_back(hash.ToString()); + + if (!include_mempool_sequence) { + return a; + } else { + UniValue o(UniValue::VOBJ); + o.pushKV("txids", a); + o.pushKV("mempool_sequence", mempool_sequence); + return o; + } + } +} + +static RPCHelpMan getrawmempool() +{ + return RPCHelpMan{"getrawmempool", + "\nReturns all transaction ids in memory pool as a json array of string transaction ids.\n" + "\nHint: use getmempoolentry to fetch a specific transaction from the mempool.\n", + { + {"verbose", RPCArg::Type::BOOL, RPCArg::Default{false}, "True for a json object, false for array of transaction ids"}, + {"mempool_sequence", RPCArg::Type::BOOL, RPCArg::Default{false}, "If verbose=false, returns a json object with transaction list and mempool sequence number attached."}, + }, + { + RPCResult{"for verbose = false", + RPCResult::Type::ARR, "", "", + { + {RPCResult::Type::STR_HEX, "", "The transaction id"}, + }}, + RPCResult{"for verbose = true", + RPCResult::Type::OBJ_DYN, "", "", + { + {RPCResult::Type::OBJ, "transactionid", "", MempoolEntryDescription()}, + }}, + RPCResult{"for verbose = false and mempool_sequence = true", + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::ARR, "txids", "", + { + {RPCResult::Type::STR_HEX, "", "The transaction id"}, + }}, + {RPCResult::Type::NUM, "mempool_sequence", "The mempool sequence value."}, + }}, + }, + RPCExamples{ + HelpExampleCli("getrawmempool", "true") + + HelpExampleRpc("getrawmempool", "true") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + bool fVerbose = false; + if (!request.params[0].isNull()) + fVerbose = request.params[0].get_bool(); + + bool include_mempool_sequence = false; + if (!request.params[1].isNull()) { + include_mempool_sequence = request.params[1].get_bool(); + } + + return MempoolToJSON(EnsureAnyMemPool(request.context), fVerbose, include_mempool_sequence); +}, + }; +} + +static RPCHelpMan getmempoolancestors() +{ + return RPCHelpMan{"getmempoolancestors", + "\nIf txid is in the mempool, returns all in-mempool ancestors.\n", + { + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id (must be in mempool)"}, + {"verbose", RPCArg::Type::BOOL, RPCArg::Default{false}, "True for a json object, false for array of transaction ids"}, + }, + { + RPCResult{"for verbose = false", + RPCResult::Type::ARR, "", "", + {{RPCResult::Type::STR_HEX, "", "The transaction id of an in-mempool ancestor transaction"}}}, + RPCResult{"for verbose = true", + RPCResult::Type::OBJ_DYN, "", "", + { + {RPCResult::Type::OBJ, "transactionid", "", MempoolEntryDescription()}, + }}, + }, + RPCExamples{ + HelpExampleCli("getmempoolancestors", "\"mytxid\"") + + HelpExampleRpc("getmempoolancestors", "\"mytxid\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + bool fVerbose = false; + if (!request.params[1].isNull()) + fVerbose = request.params[1].get_bool(); + + uint256 hash = ParseHashV(request.params[0], "parameter 1"); + + const CTxMemPool& mempool = EnsureAnyMemPool(request.context); + LOCK(mempool.cs); + + CTxMemPool::txiter it = mempool.mapTx.find(hash); + if (it == mempool.mapTx.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); + } + + CTxMemPool::setEntries setAncestors; + uint64_t noLimit = std::numeric_limits<uint64_t>::max(); + std::string dummy; + mempool.CalculateMemPoolAncestors(*it, setAncestors, noLimit, noLimit, noLimit, noLimit, dummy, false); + + if (!fVerbose) { + UniValue o(UniValue::VARR); + for (CTxMemPool::txiter ancestorIt : setAncestors) { + o.push_back(ancestorIt->GetTx().GetHash().ToString()); + } + return o; + } else { + UniValue o(UniValue::VOBJ); + for (CTxMemPool::txiter ancestorIt : setAncestors) { + const CTxMemPoolEntry &e = *ancestorIt; + const uint256& _hash = e.GetTx().GetHash(); + UniValue info(UniValue::VOBJ); + entryToJSON(mempool, info, e); + o.pushKV(_hash.ToString(), info); + } + return o; + } +}, + }; +} + +static RPCHelpMan getmempooldescendants() +{ + return RPCHelpMan{"getmempooldescendants", + "\nIf txid is in the mempool, returns all in-mempool descendants.\n", + { + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id (must be in mempool)"}, + {"verbose", RPCArg::Type::BOOL, RPCArg::Default{false}, "True for a json object, false for array of transaction ids"}, + }, + { + RPCResult{"for verbose = false", + RPCResult::Type::ARR, "", "", + {{RPCResult::Type::STR_HEX, "", "The transaction id of an in-mempool descendant transaction"}}}, + RPCResult{"for verbose = true", + RPCResult::Type::OBJ_DYN, "", "", + { + {RPCResult::Type::OBJ, "transactionid", "", MempoolEntryDescription()}, + }}, + }, + RPCExamples{ + HelpExampleCli("getmempooldescendants", "\"mytxid\"") + + HelpExampleRpc("getmempooldescendants", "\"mytxid\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + bool fVerbose = false; + if (!request.params[1].isNull()) + fVerbose = request.params[1].get_bool(); + + uint256 hash = ParseHashV(request.params[0], "parameter 1"); + + const CTxMemPool& mempool = EnsureAnyMemPool(request.context); + LOCK(mempool.cs); + + CTxMemPool::txiter it = mempool.mapTx.find(hash); + if (it == mempool.mapTx.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); + } + + CTxMemPool::setEntries setDescendants; + mempool.CalculateDescendants(it, setDescendants); + // CTxMemPool::CalculateDescendants will include the given tx + setDescendants.erase(it); + + if (!fVerbose) { + UniValue o(UniValue::VARR); + for (CTxMemPool::txiter descendantIt : setDescendants) { + o.push_back(descendantIt->GetTx().GetHash().ToString()); + } + + return o; + } else { + UniValue o(UniValue::VOBJ); + for (CTxMemPool::txiter descendantIt : setDescendants) { + const CTxMemPoolEntry &e = *descendantIt; + const uint256& _hash = e.GetTx().GetHash(); + UniValue info(UniValue::VOBJ); + entryToJSON(mempool, info, e); + o.pushKV(_hash.ToString(), info); + } + return o; + } +}, + }; +} + +static RPCHelpMan getmempoolentry() +{ + return RPCHelpMan{"getmempoolentry", + "\nReturns mempool data for given transaction\n", + { + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id (must be in mempool)"}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", MempoolEntryDescription()}, + RPCExamples{ + HelpExampleCli("getmempoolentry", "\"mytxid\"") + + HelpExampleRpc("getmempoolentry", "\"mytxid\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + uint256 hash = ParseHashV(request.params[0], "parameter 1"); + + const CTxMemPool& mempool = EnsureAnyMemPool(request.context); + LOCK(mempool.cs); + + CTxMemPool::txiter it = mempool.mapTx.find(hash); + if (it == mempool.mapTx.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); + } + + const CTxMemPoolEntry &e = *it; + UniValue info(UniValue::VOBJ); + entryToJSON(mempool, info, e); + return info; +}, + }; +} + +UniValue MempoolInfoToJSON(const CTxMemPool& pool) +{ + // Make sure this call is atomic in the pool. + LOCK(pool.cs); + UniValue ret(UniValue::VOBJ); + ret.pushKV("loaded", pool.IsLoaded()); + ret.pushKV("size", (int64_t)pool.size()); + ret.pushKV("bytes", (int64_t)pool.GetTotalTxSize()); + ret.pushKV("usage", (int64_t)pool.DynamicMemoryUsage()); + ret.pushKV("total_fee", ValueFromAmount(pool.GetTotalFee())); + int64_t maxmempool{gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000}; + ret.pushKV("maxmempool", maxmempool); + ret.pushKV("mempoolminfee", ValueFromAmount(std::max(pool.GetMinFee(maxmempool), ::minRelayTxFee).GetFeePerK())); + ret.pushKV("minrelaytxfee", ValueFromAmount(::minRelayTxFee.GetFeePerK())); + ret.pushKV("unbroadcastcount", uint64_t{pool.GetUnbroadcastTxs().size()}); + return ret; +} + +static RPCHelpMan getmempoolinfo() +{ + return RPCHelpMan{"getmempoolinfo", + "\nReturns details on the active state of the TX memory pool.\n", + {}, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::BOOL, "loaded", "True if the mempool is fully loaded"}, + {RPCResult::Type::NUM, "size", "Current tx count"}, + {RPCResult::Type::NUM, "bytes", "Sum of all virtual transaction sizes as defined in BIP 141. Differs from actual serialized size because witness data is discounted"}, + {RPCResult::Type::NUM, "usage", "Total memory usage for the mempool"}, + {RPCResult::Type::STR_AMOUNT, "total_fee", "Total fees for the mempool in " + CURRENCY_UNIT + ", ignoring modified fees through prioritisetransaction"}, + {RPCResult::Type::NUM, "maxmempool", "Maximum memory usage for the mempool"}, + {RPCResult::Type::STR_AMOUNT, "mempoolminfee", "Minimum fee rate in " + CURRENCY_UNIT + "/kvB for tx to be accepted. Is the maximum of minrelaytxfee and minimum mempool fee"}, + {RPCResult::Type::STR_AMOUNT, "minrelaytxfee", "Current minimum relay fee for transactions"}, + {RPCResult::Type::NUM, "unbroadcastcount", "Current number of transactions that haven't passed initial broadcast yet"} + }}, + RPCExamples{ + HelpExampleCli("getmempoolinfo", "") + + HelpExampleRpc("getmempoolinfo", "") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + return MempoolInfoToJSON(EnsureAnyMemPool(request.context)); +}, + }; +} + +static RPCHelpMan savemempool() +{ + return RPCHelpMan{"savemempool", + "\nDumps the mempool to disk. It will fail until the previous dump is fully loaded.\n", + {}, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR, "filename", "the directory and file where the mempool was saved"}, + }}, + RPCExamples{ + HelpExampleCli("savemempool", "") + + HelpExampleRpc("savemempool", "") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + const ArgsManager& args{EnsureAnyArgsman(request.context)}; + const CTxMemPool& mempool = EnsureAnyMemPool(request.context); + + if (!mempool.IsLoaded()) { + throw JSONRPCError(RPC_MISC_ERROR, "The mempool was not loaded yet"); + } + + if (!DumpMempool(mempool)) { + throw JSONRPCError(RPC_MISC_ERROR, "Unable to dump mempool to disk"); + } + + UniValue ret(UniValue::VOBJ); + ret.pushKV("filename", fs::path((args.GetDataDirNet() / "mempool.dat")).u8string()); + + return ret; +}, + }; +} + +void RegisterMempoolRPCCommands(CRPCTable& t) +{ + static const CRPCCommand commands[]{ + {"rawtransactions", &sendrawtransaction}, + {"rawtransactions", &testmempoolaccept}, + {"blockchain", &getmempoolancestors}, + {"blockchain", &getmempooldescendants}, + {"blockchain", &getmempoolentry}, + {"blockchain", &getmempoolinfo}, + {"blockchain", &getrawmempool}, + {"blockchain", &savemempool}, + }; + for (const auto& c : commands) { + t.appendCommand(c.name, &c); + } +} diff --git a/src/rpc/mempool.h b/src/rpc/mempool.h new file mode 100644 index 0000000000..229d7d52dd --- /dev/null +++ b/src/rpc/mempool.h @@ -0,0 +1,17 @@ +// Copyright (c) 2017-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_RPC_MEMPOOL_H +#define BITCOIN_RPC_MEMPOOL_H + +class CTxMemPool; +class UniValue; + +/** Mempool information to JSON */ +UniValue MempoolInfoToJSON(const CTxMemPool& pool); + +/** Mempool to JSON */ +UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose = false, bool include_mempool_sequence = false); + +#endif // BITCOIN_RPC_MEMPOOL_H diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 1d1ae92c58..b552528951 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -7,6 +7,7 @@ #include <chainparams.h> #include <consensus/amount.h> #include <consensus/consensus.h> +#include <consensus/merkle.h> #include <consensus/params.h> #include <consensus/validation.h> #include <core_io.h> @@ -16,7 +17,6 @@ #include <net.h> #include <node/context.h> #include <node/miner.h> -#include <policy/fees.h> #include <pow.h> #include <rpc/blockchain.h> #include <rpc/mining.h> @@ -29,7 +29,6 @@ #include <shutdown.h> #include <txmempool.h> #include <univalue.h> -#include <util/fees.h> #include <util/strencodings.h> #include <util/string.h> #include <util/system.h> @@ -43,7 +42,6 @@ using node::BlockAssembler; using node::CBlockTemplate; -using node::IncrementExtraNonce; using node::NodeContext; using node::RegenerateCommitments; using node::UpdateTime; @@ -116,14 +114,10 @@ static RPCHelpMan getnetworkhashps() }; } -static bool GenerateBlock(ChainstateManager& chainman, CBlock& block, uint64_t& max_tries, unsigned int& extra_nonce, uint256& block_hash) +static bool GenerateBlock(ChainstateManager& chainman, CBlock& block, uint64_t& max_tries, uint256& block_hash) { block_hash.SetNull(); - - { - LOCK(cs_main); - IncrementExtraNonce(&block, chainman.ActiveChain().Tip(), extra_nonce); - } + block.hashMerkleRoot = BlockMerkleRoot(block); CChainParams chainparams(Params()); @@ -149,30 +143,20 @@ static bool GenerateBlock(ChainstateManager& chainman, CBlock& block, uint64_t& static UniValue generateBlocks(ChainstateManager& chainman, const CTxMemPool& mempool, const CScript& coinbase_script, int nGenerate, uint64_t nMaxTries) { - int nHeightEnd = 0; - int nHeight = 0; - - { // Don't keep cs_main locked - LOCK(cs_main); - nHeight = chainman.ActiveChain().Height(); - nHeightEnd = nHeight+nGenerate; - } - unsigned int nExtraNonce = 0; UniValue blockHashes(UniValue::VARR); - while (nHeight < nHeightEnd && !ShutdownRequested()) - { + while (nGenerate > 0 && !ShutdownRequested()) { std::unique_ptr<CBlockTemplate> pblocktemplate(BlockAssembler(chainman.ActiveChainstate(), mempool, Params()).CreateNewBlock(coinbase_script)); if (!pblocktemplate.get()) throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block"); CBlock *pblock = &pblocktemplate->block; uint256 block_hash; - if (!GenerateBlock(chainman, *pblock, nMaxTries, nExtraNonce, block_hash)) { + if (!GenerateBlock(chainman, *pblock, nMaxTries, block_hash)) { break; } if (!block_hash.IsNull()) { - ++nHeight; + --nGenerate; blockHashes.push_back(block_hash.GetHex()); } } @@ -397,9 +381,8 @@ static RPCHelpMan generateblock() uint256 block_hash; uint64_t max_tries{DEFAULT_MAX_TRIES}; - unsigned int extra_nonce{0}; - if (!GenerateBlock(chainman, block, max_tries, extra_nonce, block_hash) || block_hash.IsNull()) { + if (!GenerateBlock(chainman, block, max_tries, block_hash) || block_hash.IsNull()) { throw JSONRPCError(RPC_MISC_ERROR, "Failed to make block."); } @@ -1069,225 +1052,21 @@ static RPCHelpMan submitheader() }; } -static RPCHelpMan estimatesmartfee() -{ - return RPCHelpMan{"estimatesmartfee", - "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" - "confirmation within conf_target blocks if possible and return the number of blocks\n" - "for which the estimate is valid. Uses virtual transaction size as defined\n" - "in BIP 141 (witness data is discounted).\n", - { - {"conf_target", RPCArg::Type::NUM, RPCArg::Optional::NO, "Confirmation target in blocks (1 - 1008)"}, - {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"conservative"}, "The fee estimate mode.\n" - " Whether to return a more conservative estimate which also satisfies\n" - " a longer history. A conservative estimate potentially returns a\n" - " higher feerate and is more likely to be sufficient for the desired\n" - " target, but is not as responsive to short term drops in the\n" - " prevailing fee market. Must be one of (case insensitive):\n" - "\"" + FeeModes("\"\n\"") + "\""}, - }, - RPCResult{ - RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::NUM, "feerate", /*optional=*/true, "estimate fee rate in " + CURRENCY_UNIT + "/kvB (only present if no errors were encountered)"}, - {RPCResult::Type::ARR, "errors", /*optional=*/true, "Errors encountered during processing (if there are any)", - { - {RPCResult::Type::STR, "", "error"}, - }}, - {RPCResult::Type::NUM, "blocks", "block number where estimate was found\n" - "The request target will be clamped between 2 and the highest target\n" - "fee estimation is able to return based on how long it has been running.\n" - "An error is returned if not enough transactions and blocks\n" - "have been observed to make an estimate for any number of blocks."}, - }}, - RPCExamples{ - HelpExampleCli("estimatesmartfee", "6") + - HelpExampleRpc("estimatesmartfee", "6") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +void RegisterMiningRPCCommands(CRPCTable& t) { - RPCTypeCheck(request.params, {UniValue::VNUM, UniValue::VSTR}); - RPCTypeCheckArgument(request.params[0], UniValue::VNUM); - - CBlockPolicyEstimator& fee_estimator = EnsureAnyFeeEstimator(request.context); - const NodeContext& node = EnsureAnyNodeContext(request.context); - const CTxMemPool& mempool = EnsureMemPool(node); - - unsigned int max_target = fee_estimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); - unsigned int conf_target = ParseConfirmTarget(request.params[0], max_target); - bool conservative = true; - if (!request.params[1].isNull()) { - FeeEstimateMode fee_mode; - if (!FeeModeFromString(request.params[1].get_str(), fee_mode)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, InvalidEstimateModeErrorMessage()); - } - if (fee_mode == FeeEstimateMode::ECONOMICAL) conservative = false; - } - - UniValue result(UniValue::VOBJ); - UniValue errors(UniValue::VARR); - FeeCalculation feeCalc; - CFeeRate feeRate{fee_estimator.estimateSmartFee(conf_target, &feeCalc, conservative)}; - if (feeRate != CFeeRate(0)) { - CFeeRate min_mempool_feerate{mempool.GetMinFee(gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000)}; - CFeeRate min_relay_feerate{::minRelayTxFee}; - feeRate = std::max({feeRate, min_mempool_feerate, min_relay_feerate}); - result.pushKV("feerate", ValueFromAmount(feeRate.GetFeePerK())); - } else { - errors.push_back("Insufficient data or no feerate found"); - result.pushKV("errors", errors); - } - result.pushKV("blocks", feeCalc.returnedTarget); - return result; -}, + static const CRPCCommand commands[]{ + {"mining", &getnetworkhashps}, + {"mining", &getmininginfo}, + {"mining", &prioritisetransaction}, + {"mining", &getblocktemplate}, + {"mining", &submitblock}, + {"mining", &submitheader}, + + {"hidden", &generatetoaddress}, + {"hidden", &generatetodescriptor}, + {"hidden", &generateblock}, + {"hidden", &generate}, }; -} - -static RPCHelpMan estimaterawfee() -{ - return RPCHelpMan{"estimaterawfee", - "\nWARNING: This interface is unstable and may disappear or change!\n" - "\nWARNING: This is an advanced API call that is tightly coupled to the specific\n" - " implementation of fee estimation. The parameters it can be called with\n" - " and the results it returns will change if the internal implementation changes.\n" - "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" - "confirmation within conf_target blocks if possible. Uses virtual transaction size as\n" - "defined in BIP 141 (witness data is discounted).\n", - { - {"conf_target", RPCArg::Type::NUM, RPCArg::Optional::NO, "Confirmation target in blocks (1 - 1008)"}, - {"threshold", RPCArg::Type::NUM, RPCArg::Default{0.95}, "The proportion of transactions in a given feerate range that must have been\n" - " confirmed within conf_target in order to consider those feerates as high enough and proceed to check\n" - " lower buckets."}, - }, - RPCResult{ - RPCResult::Type::OBJ, "", "Results are returned for any horizon which tracks blocks up to the confirmation target", - { - {RPCResult::Type::OBJ, "short", /*optional=*/true, "estimate for short time horizon", - { - {RPCResult::Type::NUM, "feerate", /*optional=*/true, "estimate fee rate in " + CURRENCY_UNIT + "/kvB"}, - {RPCResult::Type::NUM, "decay", "exponential decay (per block) for historical moving average of confirmation data"}, - {RPCResult::Type::NUM, "scale", "The resolution of confirmation targets at this time horizon"}, - {RPCResult::Type::OBJ, "pass", /*optional=*/true, "information about the lowest range of feerates to succeed in meeting the threshold", - { - {RPCResult::Type::NUM, "startrange", "start of feerate range"}, - {RPCResult::Type::NUM, "endrange", "end of feerate range"}, - {RPCResult::Type::NUM, "withintarget", "number of txs over history horizon in the feerate range that were confirmed within target"}, - {RPCResult::Type::NUM, "totalconfirmed", "number of txs over history horizon in the feerate range that were confirmed at any point"}, - {RPCResult::Type::NUM, "inmempool", "current number of txs in mempool in the feerate range unconfirmed for at least target blocks"}, - {RPCResult::Type::NUM, "leftmempool", "number of txs over history horizon in the feerate range that left mempool unconfirmed after target"}, - }}, - {RPCResult::Type::OBJ, "fail", /*optional=*/true, "information about the highest range of feerates to fail to meet the threshold", - { - {RPCResult::Type::ELISION, "", ""}, - }}, - {RPCResult::Type::ARR, "errors", /*optional=*/true, "Errors encountered during processing (if there are any)", - { - {RPCResult::Type::STR, "error", ""}, - }}, - }}, - {RPCResult::Type::OBJ, "medium", /*optional=*/true, "estimate for medium time horizon", - { - {RPCResult::Type::ELISION, "", ""}, - }}, - {RPCResult::Type::OBJ, "long", /*optional=*/true, "estimate for long time horizon", - { - {RPCResult::Type::ELISION, "", ""}, - }}, - }}, - RPCExamples{ - HelpExampleCli("estimaterawfee", "6 0.9") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - RPCTypeCheck(request.params, {UniValue::VNUM, UniValue::VNUM}, true); - RPCTypeCheckArgument(request.params[0], UniValue::VNUM); - - CBlockPolicyEstimator& fee_estimator = EnsureAnyFeeEstimator(request.context); - - unsigned int max_target = fee_estimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); - unsigned int conf_target = ParseConfirmTarget(request.params[0], max_target); - double threshold = 0.95; - if (!request.params[1].isNull()) { - threshold = request.params[1].get_real(); - } - if (threshold < 0 || threshold > 1) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid threshold"); - } - - UniValue result(UniValue::VOBJ); - - for (const FeeEstimateHorizon horizon : ALL_FEE_ESTIMATE_HORIZONS) { - CFeeRate feeRate; - EstimationResult buckets; - - // Only output results for horizons which track the target - if (conf_target > fee_estimator.HighestTargetTracked(horizon)) continue; - - feeRate = fee_estimator.estimateRawFee(conf_target, threshold, horizon, &buckets); - UniValue horizon_result(UniValue::VOBJ); - UniValue errors(UniValue::VARR); - UniValue passbucket(UniValue::VOBJ); - passbucket.pushKV("startrange", round(buckets.pass.start)); - passbucket.pushKV("endrange", round(buckets.pass.end)); - passbucket.pushKV("withintarget", round(buckets.pass.withinTarget * 100.0) / 100.0); - passbucket.pushKV("totalconfirmed", round(buckets.pass.totalConfirmed * 100.0) / 100.0); - passbucket.pushKV("inmempool", round(buckets.pass.inMempool * 100.0) / 100.0); - passbucket.pushKV("leftmempool", round(buckets.pass.leftMempool * 100.0) / 100.0); - UniValue failbucket(UniValue::VOBJ); - failbucket.pushKV("startrange", round(buckets.fail.start)); - failbucket.pushKV("endrange", round(buckets.fail.end)); - failbucket.pushKV("withintarget", round(buckets.fail.withinTarget * 100.0) / 100.0); - failbucket.pushKV("totalconfirmed", round(buckets.fail.totalConfirmed * 100.0) / 100.0); - failbucket.pushKV("inmempool", round(buckets.fail.inMempool * 100.0) / 100.0); - failbucket.pushKV("leftmempool", round(buckets.fail.leftMempool * 100.0) / 100.0); - - // CFeeRate(0) is used to indicate error as a return value from estimateRawFee - if (feeRate != CFeeRate(0)) { - horizon_result.pushKV("feerate", ValueFromAmount(feeRate.GetFeePerK())); - horizon_result.pushKV("decay", buckets.decay); - horizon_result.pushKV("scale", (int)buckets.scale); - horizon_result.pushKV("pass", passbucket); - // buckets.fail.start == -1 indicates that all buckets passed, there is no fail bucket to output - if (buckets.fail.start != -1) horizon_result.pushKV("fail", failbucket); - } else { - // Output only information that is still meaningful in the event of error - horizon_result.pushKV("decay", buckets.decay); - horizon_result.pushKV("scale", (int)buckets.scale); - horizon_result.pushKV("fail", failbucket); - errors.push_back("Insufficient data or no feerate found which meets threshold"); - horizon_result.pushKV("errors",errors); - } - result.pushKV(StringForFeeEstimateHorizon(horizon), horizon_result); - } - return result; -}, - }; -} - -void RegisterMiningRPCCommands(CRPCTable &t) -{ -// clang-format off -static const CRPCCommand commands[] = -{ // category actor (function) - // --------------------- ----------------------- - { "mining", &getnetworkhashps, }, - { "mining", &getmininginfo, }, - { "mining", &prioritisetransaction, }, - { "mining", &getblocktemplate, }, - { "mining", &submitblock, }, - { "mining", &submitheader, }, - - - { "hidden", &generatetoaddress, }, - { "hidden", &generatetodescriptor, }, - { "hidden", &generateblock, }, - - { "util", &estimatesmartfee, }, - - { "hidden", &estimaterawfee, }, - { "hidden", &generate, }, -}; -// clang-format on for (const auto& c : commands) { t.appendCommand(c.name, &c); } diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp deleted file mode 100644 index 8d7b48d697..0000000000 --- a/src/rpc/misc.cpp +++ /dev/null @@ -1,833 +0,0 @@ -// Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <httpserver.h> -#include <index/blockfilterindex.h> -#include <index/coinstatsindex.h> -#include <index/txindex.h> -#include <interfaces/chain.h> -#include <interfaces/echo.h> -#include <interfaces/init.h> -#include <interfaces/ipc.h> -#include <key_io.h> -#include <node/context.h> -#include <outputtype.h> -#include <rpc/blockchain.h> -#include <rpc/server.h> -#include <rpc/server_util.h> -#include <rpc/util.h> -#include <scheduler.h> -#include <script/descriptor.h> -#include <util/check.h> -#include <util/message.h> // For MessageSign(), MessageVerify() -#include <util/strencodings.h> -#include <util/syscall_sandbox.h> -#include <util/system.h> - -#include <optional> -#include <stdint.h> -#include <tuple> -#ifdef HAVE_MALLOC_INFO -#include <malloc.h> -#endif - -#include <univalue.h> - -using node::NodeContext; - -static RPCHelpMan validateaddress() -{ - return RPCHelpMan{ - "validateaddress", - "\nReturn information about the given bitcoin address.\n", - { - {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address to validate"}, - }, - RPCResult{ - RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::BOOL, "isvalid", "If the address is valid or not"}, - {RPCResult::Type::STR, "address", /*optional=*/true, "The bitcoin address validated"}, - {RPCResult::Type::STR_HEX, "scriptPubKey", /*optional=*/true, "The hex-encoded scriptPubKey generated by the address"}, - {RPCResult::Type::BOOL, "isscript", /*optional=*/true, "If the key is a script"}, - {RPCResult::Type::BOOL, "iswitness", /*optional=*/true, "If the address is a witness address"}, - {RPCResult::Type::NUM, "witness_version", /*optional=*/true, "The version number of the witness program"}, - {RPCResult::Type::STR_HEX, "witness_program", /*optional=*/true, "The hex value of the witness program"}, - {RPCResult::Type::STR, "error", /*optional=*/true, "Error message, if any"}, - {RPCResult::Type::ARR, "error_locations", /*optional=*/true, "Indices of likely error locations in address, if known (e.g. Bech32 errors)", - { - {RPCResult::Type::NUM, "index", "index of a potential error"}, - }}, - } - }, - RPCExamples{ - HelpExampleCli("validateaddress", "\"" + EXAMPLE_ADDRESS[0] + "\"") + - HelpExampleRpc("validateaddress", "\"" + EXAMPLE_ADDRESS[0] + "\"") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - std::string error_msg; - std::vector<int> error_locations; - CTxDestination dest = DecodeDestination(request.params[0].get_str(), error_msg, &error_locations); - const bool isValid = IsValidDestination(dest); - CHECK_NONFATAL(isValid == error_msg.empty()); - - UniValue ret(UniValue::VOBJ); - ret.pushKV("isvalid", isValid); - if (isValid) { - std::string currentAddress = EncodeDestination(dest); - ret.pushKV("address", currentAddress); - - CScript scriptPubKey = GetScriptForDestination(dest); - ret.pushKV("scriptPubKey", HexStr(scriptPubKey)); - - UniValue detail = DescribeAddress(dest); - ret.pushKVs(detail); - } else { - UniValue error_indices(UniValue::VARR); - for (int i : error_locations) error_indices.push_back(i); - ret.pushKV("error_locations", error_indices); - ret.pushKV("error", error_msg); - } - - return ret; -}, - }; -} - -static RPCHelpMan createmultisig() -{ - return RPCHelpMan{"createmultisig", - "\nCreates a multi-signature address with n signature of m keys required.\n" - "It returns a json object with the address and redeemScript.\n", - { - {"nrequired", RPCArg::Type::NUM, RPCArg::Optional::NO, "The number of required signatures out of the n keys."}, - {"keys", RPCArg::Type::ARR, RPCArg::Optional::NO, "The hex-encoded public keys.", - { - {"key", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "The hex-encoded public key"}, - }}, - {"address_type", RPCArg::Type::STR, RPCArg::Default{"legacy"}, "The address type to use. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, - }, - RPCResult{ - RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR, "address", "The value of the new multisig address."}, - {RPCResult::Type::STR_HEX, "redeemScript", "The string value of the hex-encoded redemption script."}, - {RPCResult::Type::STR, "descriptor", "The descriptor for this multisig"}, - {RPCResult::Type::ARR, "warnings", /* optional */ true, "Any warnings resulting from the creation of this multisig", - { - {RPCResult::Type::STR, "", ""}, - }}, - } - }, - RPCExamples{ - "\nCreate a multisig address from 2 public keys\n" - + HelpExampleCli("createmultisig", "2 \"[\\\"03789ed0bb717d88f7d321a368d905e7430207ebbd82bd342cf11ae157a7ace5fd\\\",\\\"03dbc6764b8884a92e871274b87583e6d5c2a58819473e17e107ef3f6aa5a61626\\\"]\"") + - "\nAs a JSON-RPC call\n" - + HelpExampleRpc("createmultisig", "2, [\"03789ed0bb717d88f7d321a368d905e7430207ebbd82bd342cf11ae157a7ace5fd\",\"03dbc6764b8884a92e871274b87583e6d5c2a58819473e17e107ef3f6aa5a61626\"]") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - int required = request.params[0].get_int(); - - // Get the public keys - const UniValue& keys = request.params[1].get_array(); - std::vector<CPubKey> pubkeys; - for (unsigned int i = 0; i < keys.size(); ++i) { - if (IsHex(keys[i].get_str()) && (keys[i].get_str().length() == 66 || keys[i].get_str().length() == 130)) { - pubkeys.push_back(HexToPubKey(keys[i].get_str())); - } else { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Invalid public key: %s\n.", keys[i].get_str())); - } - } - - // Get the output type - OutputType output_type = OutputType::LEGACY; - if (!request.params[2].isNull()) { - std::optional<OutputType> parsed = ParseOutputType(request.params[2].get_str()); - if (!parsed) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Unknown address type '%s'", request.params[2].get_str())); - } else if (parsed.value() == OutputType::BECH32M) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "createmultisig cannot create bech32m multisig addresses"); - } - output_type = parsed.value(); - } - - // Construct using pay-to-script-hash: - FillableSigningProvider keystore; - CScript inner; - const CTxDestination dest = AddAndGetMultisigDestination(required, pubkeys, output_type, keystore, inner); - - // Make the descriptor - std::unique_ptr<Descriptor> descriptor = InferDescriptor(GetScriptForDestination(dest), keystore); - - UniValue result(UniValue::VOBJ); - result.pushKV("address", EncodeDestination(dest)); - result.pushKV("redeemScript", HexStr(inner)); - result.pushKV("descriptor", descriptor->ToString()); - - UniValue warnings(UniValue::VARR); - if (!request.params[2].isNull() && OutputTypeFromDestination(dest) != output_type) { - // Only warns if the user has explicitly chosen an address type we cannot generate - warnings.push_back("Unable to make chosen address type, please ensure no uncompressed public keys are present."); - } - if (warnings.size()) result.pushKV("warnings", warnings); - - return result; -}, - }; -} - -static RPCHelpMan getdescriptorinfo() -{ - const std::string EXAMPLE_DESCRIPTOR = "wpkh([d34db33f/84h/0h/0h]0279be667ef9dcbbac55a06295Ce870b07029Bfcdb2dce28d959f2815b16f81798)"; - - return RPCHelpMan{"getdescriptorinfo", - {"\nAnalyses a descriptor.\n"}, - { - {"descriptor", RPCArg::Type::STR, RPCArg::Optional::NO, "The descriptor."}, - }, - RPCResult{ - RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR, "descriptor", "The descriptor in canonical form, without private keys"}, - {RPCResult::Type::STR, "checksum", "The checksum for the input descriptor"}, - {RPCResult::Type::BOOL, "isrange", "Whether the descriptor is ranged"}, - {RPCResult::Type::BOOL, "issolvable", "Whether the descriptor is solvable"}, - {RPCResult::Type::BOOL, "hasprivatekeys", "Whether the input descriptor contained at least one private key"}, - } - }, - RPCExamples{ - "Analyse a descriptor\n" + - HelpExampleCli("getdescriptorinfo", "\"" + EXAMPLE_DESCRIPTOR + "\"") + - HelpExampleRpc("getdescriptorinfo", "\"" + EXAMPLE_DESCRIPTOR + "\"") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - RPCTypeCheck(request.params, {UniValue::VSTR}); - - FlatSigningProvider provider; - std::string error; - auto desc = Parse(request.params[0].get_str(), provider, error); - if (!desc) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, error); - } - - UniValue result(UniValue::VOBJ); - result.pushKV("descriptor", desc->ToString()); - result.pushKV("checksum", GetDescriptorChecksum(request.params[0].get_str())); - result.pushKV("isrange", desc->IsRange()); - result.pushKV("issolvable", desc->IsSolvable()); - result.pushKV("hasprivatekeys", provider.keys.size() > 0); - return result; -}, - }; -} - -static RPCHelpMan deriveaddresses() -{ - const std::string EXAMPLE_DESCRIPTOR = "wpkh([d34db33f/84h/0h/0h]xpub6DJ2dNUysrn5Vt36jH2KLBT2i1auw1tTSSomg8PhqNiUtx8QX2SvC9nrHu81fT41fvDUnhMjEzQgXnQjKEu3oaqMSzhSrHMxyyoEAmUHQbY/0/*)#cjjspncu"; - - return RPCHelpMan{"deriveaddresses", - {"\nDerives one or more addresses corresponding to an output descriptor.\n" - "Examples of output descriptors are:\n" - " pkh(<pubkey>) P2PKH outputs for the given pubkey\n" - " wpkh(<pubkey>) Native segwit P2PKH outputs for the given pubkey\n" - " sh(multi(<n>,<pubkey>,<pubkey>,...)) P2SH-multisig outputs for the given threshold and pubkeys\n" - " raw(<hex script>) Outputs whose scriptPubKey equals the specified hex scripts\n" - "\nIn the above, <pubkey> either refers to a fixed public key in hexadecimal notation, or to an xpub/xprv optionally followed by one\n" - "or more path elements separated by \"/\", where \"h\" represents a hardened child key.\n" - "For more information on output descriptors, see the documentation in the doc/descriptors.md file.\n"}, - { - {"descriptor", RPCArg::Type::STR, RPCArg::Optional::NO, "The descriptor."}, - {"range", RPCArg::Type::RANGE, RPCArg::Optional::OMITTED_NAMED_ARG, "If a ranged descriptor is used, this specifies the end or the range (in [begin,end] notation) to derive."}, - }, - RPCResult{ - RPCResult::Type::ARR, "", "", - { - {RPCResult::Type::STR, "address", "the derived addresses"}, - } - }, - RPCExamples{ - "First three native segwit receive addresses\n" + - HelpExampleCli("deriveaddresses", "\"" + EXAMPLE_DESCRIPTOR + "\" \"[0,2]\"") + - HelpExampleRpc("deriveaddresses", "\"" + EXAMPLE_DESCRIPTOR + "\", \"[0,2]\"") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - RPCTypeCheck(request.params, {UniValue::VSTR, UniValueType()}); // Range argument is checked later - const std::string desc_str = request.params[0].get_str(); - - int64_t range_begin = 0; - int64_t range_end = 0; - - if (request.params.size() >= 2 && !request.params[1].isNull()) { - std::tie(range_begin, range_end) = ParseDescriptorRange(request.params[1]); - } - - FlatSigningProvider key_provider; - std::string error; - auto desc = Parse(desc_str, key_provider, error, /* require_checksum = */ true); - if (!desc) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, error); - } - - if (!desc->IsRange() && request.params.size() > 1) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Range should not be specified for an un-ranged descriptor"); - } - - if (desc->IsRange() && request.params.size() == 1) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Range must be specified for a ranged descriptor"); - } - - UniValue addresses(UniValue::VARR); - - for (int i = range_begin; i <= range_end; ++i) { - FlatSigningProvider provider; - std::vector<CScript> scripts; - if (!desc->Expand(i, key_provider, scripts, provider)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Cannot derive script without private keys"); - } - - for (const CScript &script : scripts) { - CTxDestination dest; - if (!ExtractDestination(script, dest)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Descriptor does not have a corresponding address"); - } - - addresses.push_back(EncodeDestination(dest)); - } - } - - // This should not be possible, but an assert seems overkill: - if (addresses.empty()) { - throw JSONRPCError(RPC_MISC_ERROR, "Unexpected empty result"); - } - - return addresses; -}, - }; -} - -static RPCHelpMan verifymessage() -{ - return RPCHelpMan{"verifymessage", - "Verify a signed message.", - { - {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address to use for the signature."}, - {"signature", RPCArg::Type::STR, RPCArg::Optional::NO, "The signature provided by the signer in base 64 encoding (see signmessage)."}, - {"message", RPCArg::Type::STR, RPCArg::Optional::NO, "The message that was signed."}, - }, - RPCResult{ - RPCResult::Type::BOOL, "", "If the signature is verified or not." - }, - RPCExamples{ - "\nUnlock the wallet for 30 seconds\n" - + HelpExampleCli("walletpassphrase", "\"mypassphrase\" 30") + - "\nCreate the signature\n" - + HelpExampleCli("signmessage", "\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\" \"my message\"") + - "\nVerify the signature\n" - + HelpExampleCli("verifymessage", "\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\" \"signature\" \"my message\"") + - "\nAs a JSON-RPC call\n" - + HelpExampleRpc("verifymessage", "\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\", \"signature\", \"my message\"") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - LOCK(cs_main); - - std::string strAddress = request.params[0].get_str(); - std::string strSign = request.params[1].get_str(); - std::string strMessage = request.params[2].get_str(); - - switch (MessageVerify(strAddress, strSign, strMessage)) { - case MessageVerificationResult::ERR_INVALID_ADDRESS: - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); - case MessageVerificationResult::ERR_ADDRESS_NO_KEY: - throw JSONRPCError(RPC_TYPE_ERROR, "Address does not refer to key"); - case MessageVerificationResult::ERR_MALFORMED_SIGNATURE: - throw JSONRPCError(RPC_TYPE_ERROR, "Malformed base64 encoding"); - case MessageVerificationResult::ERR_PUBKEY_NOT_RECOVERED: - case MessageVerificationResult::ERR_NOT_SIGNED: - return false; - case MessageVerificationResult::OK: - return true; - } - - return false; -}, - }; -} - -static RPCHelpMan signmessagewithprivkey() -{ - return RPCHelpMan{"signmessagewithprivkey", - "\nSign a message with the private key of an address\n", - { - {"privkey", RPCArg::Type::STR, RPCArg::Optional::NO, "The private key to sign the message with."}, - {"message", RPCArg::Type::STR, RPCArg::Optional::NO, "The message to create a signature of."}, - }, - RPCResult{ - RPCResult::Type::STR, "signature", "The signature of the message encoded in base 64" - }, - RPCExamples{ - "\nCreate the signature\n" - + HelpExampleCli("signmessagewithprivkey", "\"privkey\" \"my message\"") + - "\nVerify the signature\n" - + HelpExampleCli("verifymessage", "\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\" \"signature\" \"my message\"") + - "\nAs a JSON-RPC call\n" - + HelpExampleRpc("signmessagewithprivkey", "\"privkey\", \"my message\"") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - std::string strPrivkey = request.params[0].get_str(); - std::string strMessage = request.params[1].get_str(); - - CKey key = DecodeSecret(strPrivkey); - if (!key.IsValid()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key"); - } - - std::string signature; - - if (!MessageSign(key, strMessage, signature)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Sign failed"); - } - - return signature; -}, - }; -} - -static RPCHelpMan setmocktime() -{ - return RPCHelpMan{"setmocktime", - "\nSet the local time to given timestamp (-regtest only)\n", - { - {"timestamp", RPCArg::Type::NUM, RPCArg::Optional::NO, UNIX_EPOCH_TIME + "\n" - "Pass 0 to go back to using the system time."}, - }, - RPCResult{RPCResult::Type::NONE, "", ""}, - RPCExamples{""}, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - if (!Params().IsMockableChain()) { - throw std::runtime_error("setmocktime is for regression testing (-regtest mode) only"); - } - - // For now, don't change mocktime if we're in the middle of validation, as - // this could have an effect on mempool time-based eviction, as well as - // IsCurrentForFeeEstimation() and IsInitialBlockDownload(). - // TODO: figure out the right way to synchronize around mocktime, and - // ensure all call sites of GetTime() are accessing this safely. - LOCK(cs_main); - - RPCTypeCheck(request.params, {UniValue::VNUM}); - const int64_t time{request.params[0].get_int64()}; - if (time < 0) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Mocktime cannot be negative: %s.", time)); - } - SetMockTime(time); - auto node_context = util::AnyPtr<NodeContext>(request.context); - if (node_context) { - for (const auto& chain_client : node_context->chain_clients) { - chain_client->setMockTime(time); - } - } - - return NullUniValue; -}, - }; -} - -#if defined(USE_SYSCALL_SANDBOX) -static RPCHelpMan invokedisallowedsyscall() -{ - return RPCHelpMan{ - "invokedisallowedsyscall", - "\nInvoke a disallowed syscall to trigger a syscall sandbox violation. Used for testing purposes.\n", - {}, - RPCResult{RPCResult::Type::NONE, "", ""}, - RPCExamples{ - HelpExampleCli("invokedisallowedsyscall", "") + HelpExampleRpc("invokedisallowedsyscall", "")}, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - if (!Params().IsTestChain()) { - throw std::runtime_error("invokedisallowedsyscall is used for testing only."); - } - TestDisallowedSandboxCall(); - return NullUniValue; - }, - }; -} -#endif // USE_SYSCALL_SANDBOX - -static RPCHelpMan mockscheduler() -{ - return RPCHelpMan{"mockscheduler", - "\nBump the scheduler into the future (-regtest only)\n", - { - {"delta_time", RPCArg::Type::NUM, RPCArg::Optional::NO, "Number of seconds to forward the scheduler into the future." }, - }, - RPCResult{RPCResult::Type::NONE, "", ""}, - RPCExamples{""}, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - if (!Params().IsMockableChain()) { - throw std::runtime_error("mockscheduler is for regression testing (-regtest mode) only"); - } - - // check params are valid values - RPCTypeCheck(request.params, {UniValue::VNUM}); - int64_t delta_seconds = request.params[0].get_int64(); - if (delta_seconds <= 0 || delta_seconds > 3600) { - throw std::runtime_error("delta_time must be between 1 and 3600 seconds (1 hr)"); - } - - auto node_context = util::AnyPtr<NodeContext>(request.context); - // protect against null pointer dereference - CHECK_NONFATAL(node_context); - CHECK_NONFATAL(node_context->scheduler); - node_context->scheduler->MockForward(std::chrono::seconds(delta_seconds)); - - return NullUniValue; -}, - }; -} - -static UniValue RPCLockedMemoryInfo() -{ - LockedPool::Stats stats = LockedPoolManager::Instance().stats(); - UniValue obj(UniValue::VOBJ); - obj.pushKV("used", uint64_t(stats.used)); - obj.pushKV("free", uint64_t(stats.free)); - obj.pushKV("total", uint64_t(stats.total)); - obj.pushKV("locked", uint64_t(stats.locked)); - obj.pushKV("chunks_used", uint64_t(stats.chunks_used)); - obj.pushKV("chunks_free", uint64_t(stats.chunks_free)); - return obj; -} - -#ifdef HAVE_MALLOC_INFO -static std::string RPCMallocInfo() -{ - char *ptr = nullptr; - size_t size = 0; - FILE *f = open_memstream(&ptr, &size); - if (f) { - malloc_info(0, f); - fclose(f); - if (ptr) { - std::string rv(ptr, size); - free(ptr); - return rv; - } - } - return ""; -} -#endif - -static RPCHelpMan getmemoryinfo() -{ - /* Please, avoid using the word "pool" here in the RPC interface or help, - * as users will undoubtedly confuse it with the other "memory pool" - */ - return RPCHelpMan{"getmemoryinfo", - "Returns an object containing information about memory usage.\n", - { - {"mode", RPCArg::Type::STR, RPCArg::Default{"stats"}, "determines what kind of information is returned.\n" - " - \"stats\" returns general statistics about memory usage in the daemon.\n" - " - \"mallocinfo\" returns an XML string describing low-level heap state (only available if compiled with glibc 2.10+)."}, - }, - { - RPCResult{"mode \"stats\"", - RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::OBJ, "locked", "Information about locked memory manager", - { - {RPCResult::Type::NUM, "used", "Number of bytes used"}, - {RPCResult::Type::NUM, "free", "Number of bytes available in current arenas"}, - {RPCResult::Type::NUM, "total", "Total number of bytes managed"}, - {RPCResult::Type::NUM, "locked", "Amount of bytes that succeeded locking. If this number is smaller than total, locking pages failed at some point and key data could be swapped to disk."}, - {RPCResult::Type::NUM, "chunks_used", "Number allocated chunks"}, - {RPCResult::Type::NUM, "chunks_free", "Number unused chunks"}, - }}, - } - }, - RPCResult{"mode \"mallocinfo\"", - RPCResult::Type::STR, "", "\"<malloc version=\"1\">...\"" - }, - }, - RPCExamples{ - HelpExampleCli("getmemoryinfo", "") - + HelpExampleRpc("getmemoryinfo", "") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - std::string mode = request.params[0].isNull() ? "stats" : request.params[0].get_str(); - if (mode == "stats") { - UniValue obj(UniValue::VOBJ); - obj.pushKV("locked", RPCLockedMemoryInfo()); - return obj; - } else if (mode == "mallocinfo") { -#ifdef HAVE_MALLOC_INFO - return RPCMallocInfo(); -#else - throw JSONRPCError(RPC_INVALID_PARAMETER, "mallocinfo mode not available"); -#endif - } else { - throw JSONRPCError(RPC_INVALID_PARAMETER, "unknown mode " + mode); - } -}, - }; -} - -static void EnableOrDisableLogCategories(UniValue cats, bool enable) { - cats = cats.get_array(); - for (unsigned int i = 0; i < cats.size(); ++i) { - std::string cat = cats[i].get_str(); - - bool success; - if (enable) { - success = LogInstance().EnableCategory(cat); - } else { - success = LogInstance().DisableCategory(cat); - } - - if (!success) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "unknown logging category " + cat); - } - } -} - -static RPCHelpMan logging() -{ - return RPCHelpMan{"logging", - "Gets and sets the logging configuration.\n" - "When called without an argument, returns the list of categories with status that are currently being debug logged or not.\n" - "When called with arguments, adds or removes categories from debug logging and return the lists above.\n" - "The arguments are evaluated in order \"include\", \"exclude\".\n" - "If an item is both included and excluded, it will thus end up being excluded.\n" - "The valid logging categories are: " + LogInstance().LogCategoriesString() + "\n" - "In addition, the following are available as category names with special meanings:\n" - " - \"all\", \"1\" : represent all logging categories.\n" - " - \"none\", \"0\" : even if other logging categories are specified, ignore all of them.\n" - , - { - {"include", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "The categories to add to debug logging", - { - {"include_category", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "the valid logging category"}, - }}, - {"exclude", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "The categories to remove from debug logging", - { - {"exclude_category", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "the valid logging category"}, - }}, - }, - RPCResult{ - RPCResult::Type::OBJ_DYN, "", "keys are the logging categories, and values indicates its status", - { - {RPCResult::Type::BOOL, "category", "if being debug logged or not. false:inactive, true:active"}, - } - }, - RPCExamples{ - HelpExampleCli("logging", "\"[\\\"all\\\"]\" \"[\\\"http\\\"]\"") - + HelpExampleRpc("logging", "[\"all\"], [\"libevent\"]") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - uint32_t original_log_categories = LogInstance().GetCategoryMask(); - if (request.params[0].isArray()) { - EnableOrDisableLogCategories(request.params[0], true); - } - if (request.params[1].isArray()) { - EnableOrDisableLogCategories(request.params[1], false); - } - uint32_t updated_log_categories = LogInstance().GetCategoryMask(); - uint32_t changed_log_categories = original_log_categories ^ updated_log_categories; - - // Update libevent logging if BCLog::LIBEVENT has changed. - // If the library version doesn't allow it, UpdateHTTPServerLogging() returns false, - // in which case we should clear the BCLog::LIBEVENT flag. - // Throw an error if the user has explicitly asked to change only the libevent - // flag and it failed. - if (changed_log_categories & BCLog::LIBEVENT) { - if (!UpdateHTTPServerLogging(LogInstance().WillLogCategory(BCLog::LIBEVENT))) { - LogInstance().DisableCategory(BCLog::LIBEVENT); - if (changed_log_categories == BCLog::LIBEVENT) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "libevent logging cannot be updated when using libevent before v2.1.1."); - } - } - } - - UniValue result(UniValue::VOBJ); - for (const auto& logCatActive : LogInstance().LogCategoriesList()) { - result.pushKV(logCatActive.category, logCatActive.active); - } - - return result; -}, - }; -} - -static RPCHelpMan echo(const std::string& name) -{ - return RPCHelpMan{name, - "\nSimply echo back the input arguments. This command is for testing.\n" - "\nIt will return an internal bug report when arg9='trigger_internal_bug' is passed.\n" - "\nThe difference between echo and echojson is that echojson has argument conversion enabled in the client-side table in " - "bitcoin-cli and the GUI. There is no server-side difference.", - { - {"arg0", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, - {"arg1", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, - {"arg2", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, - {"arg3", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, - {"arg4", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, - {"arg5", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, - {"arg6", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, - {"arg7", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, - {"arg8", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, - {"arg9", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, - }, - RPCResult{RPCResult::Type::ANY, "", "Returns whatever was passed in"}, - RPCExamples{""}, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - if (request.params[9].isStr()) { - CHECK_NONFATAL(request.params[9].get_str() != "trigger_internal_bug"); - } - - return request.params; -}, - }; -} - -static RPCHelpMan echo() { return echo("echo"); } -static RPCHelpMan echojson() { return echo("echojson"); } - -static RPCHelpMan echoipc() -{ - return RPCHelpMan{ - "echoipc", - "\nEcho back the input argument, passing it through a spawned process in a multiprocess build.\n" - "This command is for testing.\n", - {{"arg", RPCArg::Type::STR, RPCArg::Optional::NO, "The string to echo",}}, - RPCResult{RPCResult::Type::STR, "echo", "The echoed string."}, - RPCExamples{HelpExampleCli("echo", "\"Hello world\"") + - HelpExampleRpc("echo", "\"Hello world\"")}, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - interfaces::Init& local_init = *EnsureAnyNodeContext(request.context).init; - std::unique_ptr<interfaces::Echo> echo; - if (interfaces::Ipc* ipc = local_init.ipc()) { - // Spawn a new bitcoin-node process and call makeEcho to get a - // client pointer to a interfaces::Echo instance running in - // that process. This is just for testing. A slightly more - // realistic test spawning a different executable instead of - // the same executable would add a new bitcoin-echo executable, - // and spawn bitcoin-echo below instead of bitcoin-node. But - // using bitcoin-node avoids the need to build and install a - // new executable just for this one test. - auto init = ipc->spawnProcess("bitcoin-node"); - echo = init->makeEcho(); - ipc->addCleanup(*echo, [init = init.release()] { delete init; }); - } else { - // IPC support is not available because this is a bitcoind - // process not a bitcoind-node process, so just create a local - // interfaces::Echo object and return it so the `echoipc` RPC - // method will work, and the python test calling `echoipc` - // can expect the same result. - echo = local_init.makeEcho(); - } - return echo->echo(request.params[0].get_str()); - }, - }; -} - -static UniValue SummaryToJSON(const IndexSummary&& summary, std::string index_name) -{ - UniValue ret_summary(UniValue::VOBJ); - if (!index_name.empty() && index_name != summary.name) return ret_summary; - - UniValue entry(UniValue::VOBJ); - entry.pushKV("synced", summary.synced); - entry.pushKV("best_block_height", summary.best_block_height); - ret_summary.pushKV(summary.name, entry); - return ret_summary; -} - -static RPCHelpMan getindexinfo() -{ - return RPCHelpMan{"getindexinfo", - "\nReturns the status of one or all available indices currently running in the node.\n", - { - {"index_name", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "Filter results for an index with a specific name."}, - }, - RPCResult{ - RPCResult::Type::OBJ_DYN, "", "", { - { - RPCResult::Type::OBJ, "name", "The name of the index", - { - {RPCResult::Type::BOOL, "synced", "Whether the index is synced or not"}, - {RPCResult::Type::NUM, "best_block_height", "The block height to which the index is synced"}, - } - }, - }, - }, - RPCExamples{ - HelpExampleCli("getindexinfo", "") - + HelpExampleRpc("getindexinfo", "") - + HelpExampleCli("getindexinfo", "txindex") - + HelpExampleRpc("getindexinfo", "txindex") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - UniValue result(UniValue::VOBJ); - const std::string index_name = request.params[0].isNull() ? "" : request.params[0].get_str(); - - if (g_txindex) { - result.pushKVs(SummaryToJSON(g_txindex->GetSummary(), index_name)); - } - - if (g_coin_stats_index) { - result.pushKVs(SummaryToJSON(g_coin_stats_index->GetSummary(), index_name)); - } - - ForEachBlockFilterIndex([&result, &index_name](const BlockFilterIndex& index) { - result.pushKVs(SummaryToJSON(index.GetSummary(), index_name)); - }); - - return result; -}, - }; -} - -void RegisterMiscRPCCommands(CRPCTable &t) -{ -// clang-format off -static const CRPCCommand commands[] = -{ // category actor (function) - // --------------------- ------------------------ - { "control", &getmemoryinfo, }, - { "control", &logging, }, - { "util", &validateaddress, }, - { "util", &createmultisig, }, - { "util", &deriveaddresses, }, - { "util", &getdescriptorinfo, }, - { "util", &verifymessage, }, - { "util", &signmessagewithprivkey, }, - { "util", &getindexinfo, }, - - /* Not shown in help */ - { "hidden", &setmocktime, }, - { "hidden", &mockscheduler, }, - { "hidden", &echo, }, - { "hidden", &echojson, }, - { "hidden", &echoipc, }, -#if defined(USE_SYSCALL_SANDBOX) - { "hidden", &invokedisallowedsyscall, }, -#endif // USE_SYSCALL_SANDBOX -}; -// clang-format on - for (const auto& c : commands) { - t.appendCommand(c.name, &c); - } -} diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 1bde4fccbb..09dc8eb3eb 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -93,7 +93,7 @@ static RPCHelpMan getpeerinfo() { return RPCHelpMan{ "getpeerinfo", - "\nReturns data about each connected network node as a json array of objects.\n", + "Returns data about each connected network peer as a json array of objects.", {}, RPCResult{ RPCResult::Type::ARR, "", "", @@ -105,7 +105,7 @@ static RPCHelpMan getpeerinfo() {RPCResult::Type::STR, "addr", "(host:port) The IP address and port of the peer"}, {RPCResult::Type::STR, "addrbind", /*optional=*/true, "(ip:port) Bind address of the connection to the peer"}, {RPCResult::Type::STR, "addrlocal", /*optional=*/true, "(ip:port) Local address as reported by the peer"}, - {RPCResult::Type::STR, "network", "Network (" + Join(GetNetworkNames(/* append_unroutable */ true), ", ") + ")"}, + {RPCResult::Type::STR, "network", "Network (" + Join(GetNetworkNames(/*append_unroutable=*/true), ", ") + ")"}, {RPCResult::Type::NUM, "mapped_as", /*optional=*/true, "The AS in the BGP route to the peer used for diversifying\n" "peer selection (only available if the asmap config flag is set)"}, {RPCResult::Type::STR_HEX, "services", "The services offered"}, @@ -113,7 +113,7 @@ static RPCHelpMan getpeerinfo() { {RPCResult::Type::STR, "SERVICE_NAME", "the service name if it is recognised"} }}, - {RPCResult::Type::BOOL, "relaytxes", "Whether peer has asked us to relay transactions to it"}, + {RPCResult::Type::BOOL, "relaytxes", /*optional=*/true, "Whether peer has asked us to relay transactions to it"}, {RPCResult::Type::NUM_TIME, "lastsend", "The " + UNIX_EPOCH_TIME + " of the last send"}, {RPCResult::Type::NUM_TIME, "lastrecv", "The " + UNIX_EPOCH_TIME + " of the last receive"}, {RPCResult::Type::NUM_TIME, "last_transaction", "The " + UNIX_EPOCH_TIME + " of the last valid transaction received from this peer"}, @@ -144,7 +144,7 @@ static RPCHelpMan getpeerinfo() { {RPCResult::Type::STR, "permission_type", Join(NET_PERMISSIONS_DOC, ",\n") + ".\n"}, }}, - {RPCResult::Type::NUM, "minfeefilter", "The minimum fee rate for transactions this peer accepts"}, + {RPCResult::Type::NUM, "minfeefilter", /*optional=*/true, "The minimum fee rate for transactions this peer accepts"}, {RPCResult::Type::OBJ_DYN, "bytessent_per_msg", "", { {RPCResult::Type::NUM, "msg", "The total bytes sent aggregated by message type\n" @@ -156,7 +156,7 @@ static RPCHelpMan getpeerinfo() {RPCResult::Type::NUM, "msg", "The total bytes received aggregated by message type\n" "When a message type is not listed in this json object, the bytes received are 0.\n" "Only known message types can appear as keys in the object and all bytes received\n" - "of unknown message types are listed under '"+NET_MESSAGE_COMMAND_OTHER+"'."} + "of unknown message types are listed under '"+NET_MESSAGE_TYPE_OTHER+"'."} }}, {RPCResult::Type::STR, "connection_type", "Type of connection: \n" + Join(CONNECTION_TYPE_DOC, ",\n") + ".\n" "Please note this output is unlikely to be stable in upcoming releases as we iterate to\n" @@ -197,7 +197,6 @@ static RPCHelpMan getpeerinfo() } obj.pushKV("services", strprintf("%016x", stats.nServices)); obj.pushKV("servicesnames", GetServicesNames(stats.nServices)); - obj.pushKV("relaytxes", stats.fRelayTxes); obj.pushKV("lastsend", count_seconds(stats.m_last_send)); obj.pushKV("lastrecv", count_seconds(stats.m_last_recv)); obj.pushKV("last_transaction", count_seconds(stats.m_last_tx_time)); @@ -232,6 +231,8 @@ static RPCHelpMan getpeerinfo() heights.push_back(height); } obj.pushKV("inflight", heights); + obj.pushKV("relaytxes", statestats.m_relay_txs); + obj.pushKV("minfeefilter", ValueFromAmount(statestats.m_fee_filter_received)); obj.pushKV("addr_relay_enabled", statestats.m_addr_relay_enabled); obj.pushKV("addr_processed", statestats.m_addr_processed); obj.pushKV("addr_rate_limited", statestats.m_addr_rate_limited); @@ -241,21 +242,20 @@ static RPCHelpMan getpeerinfo() permissions.push_back(permission); } obj.pushKV("permissions", permissions); - obj.pushKV("minfeefilter", ValueFromAmount(stats.minFeeFilter)); - UniValue sendPerMsgCmd(UniValue::VOBJ); - for (const auto& i : stats.mapSendBytesPerMsgCmd) { + UniValue sendPerMsgType(UniValue::VOBJ); + for (const auto& i : stats.mapSendBytesPerMsgType) { if (i.second > 0) - sendPerMsgCmd.pushKV(i.first, i.second); + sendPerMsgType.pushKV(i.first, i.second); } - obj.pushKV("bytessent_per_msg", sendPerMsgCmd); + obj.pushKV("bytessent_per_msg", sendPerMsgType); - UniValue recvPerMsgCmd(UniValue::VOBJ); - for (const auto& i : stats.mapRecvBytesPerMsgCmd) { + UniValue recvPerMsgType(UniValue::VOBJ); + for (const auto& i : stats.mapRecvBytesPerMsgType) { if (i.second > 0) - recvPerMsgCmd.pushKV(i.first, i.second); + recvPerMsgType.pushKV(i.first, i.second); } - obj.pushKV("bytesrecv_per_msg", recvPerMsgCmd); + obj.pushKV("bytesrecv_per_msg", recvPerMsgType); obj.pushKV("connection_type", ConnectionTypeAsString(stats.m_conn_type)); ret.push_back(obj); @@ -888,7 +888,7 @@ static RPCHelpMan getnodeaddresses() } // returns a shuffled list of CAddress - const std::vector<CAddress> vAddr{connman.GetAddresses(count, /* max_pct */ 0, network)}; + const std::vector<CAddress> vAddr{connman.GetAddresses(count, /*max_pct=*/0, network)}; UniValue ret(UniValue::VARR); for (const CAddress& addr : vAddr) { @@ -959,30 +959,25 @@ static RPCHelpMan addpeeraddress() }; } -void RegisterNetRPCCommands(CRPCTable &t) -{ -// clang-format off -static const CRPCCommand commands[] = -{ // category actor - // --------------------- ----------------------- - { "network", &getconnectioncount, }, - { "network", &ping, }, - { "network", &getpeerinfo, }, - { "network", &addnode, }, - { "network", &disconnectnode, }, - { "network", &getaddednodeinfo, }, - { "network", &getnettotals, }, - { "network", &getnetworkinfo, }, - { "network", &setban, }, - { "network", &listbanned, }, - { "network", &clearbanned, }, - { "network", &setnetworkactive, }, - { "network", &getnodeaddresses, }, - - { "hidden", &addconnection, }, - { "hidden", &addpeeraddress, }, -}; -// clang-format on +void RegisterNetRPCCommands(CRPCTable& t) +{ + static const CRPCCommand commands[]{ + {"network", &getconnectioncount}, + {"network", &ping}, + {"network", &getpeerinfo}, + {"network", &addnode}, + {"network", &disconnectnode}, + {"network", &getaddednodeinfo}, + {"network", &getnettotals}, + {"network", &getnetworkinfo}, + {"network", &setban}, + {"network", &listbanned}, + {"network", &clearbanned}, + {"network", &setnetworkactive}, + {"network", &getnodeaddresses}, + {"hidden", &addconnection}, + {"hidden", &addpeeraddress}, + }; for (const auto& c : commands) { t.appendCommand(c.name, &c); } diff --git a/src/rpc/node.cpp b/src/rpc/node.cpp new file mode 100644 index 0000000000..24697a606e --- /dev/null +++ b/src/rpc/node.cpp @@ -0,0 +1,440 @@ +// Copyright (c) 2010 Satoshi Nakamoto +// Copyright (c) 2009-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <httpserver.h> +#include <index/blockfilterindex.h> +#include <index/coinstatsindex.h> +#include <index/txindex.h> +#include <interfaces/chain.h> +#include <interfaces/echo.h> +#include <interfaces/init.h> +#include <interfaces/ipc.h> +#include <node/context.h> +#include <rpc/server.h> +#include <rpc/server_util.h> +#include <rpc/util.h> +#include <scheduler.h> +#include <univalue.h> +#include <util/check.h> +#include <util/syscall_sandbox.h> +#include <util/system.h> + +#include <stdint.h> +#ifdef HAVE_MALLOC_INFO +#include <malloc.h> +#endif + +using node::NodeContext; + +static RPCHelpMan setmocktime() +{ + return RPCHelpMan{"setmocktime", + "\nSet the local time to given timestamp (-regtest only)\n", + { + {"timestamp", RPCArg::Type::NUM, RPCArg::Optional::NO, UNIX_EPOCH_TIME + "\n" + "Pass 0 to go back to using the system time."}, + }, + RPCResult{RPCResult::Type::NONE, "", ""}, + RPCExamples{""}, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + if (!Params().IsMockableChain()) { + throw std::runtime_error("setmocktime is for regression testing (-regtest mode) only"); + } + + // For now, don't change mocktime if we're in the middle of validation, as + // this could have an effect on mempool time-based eviction, as well as + // IsCurrentForFeeEstimation() and IsInitialBlockDownload(). + // TODO: figure out the right way to synchronize around mocktime, and + // ensure all call sites of GetTime() are accessing this safely. + LOCK(cs_main); + + RPCTypeCheck(request.params, {UniValue::VNUM}); + const int64_t time{request.params[0].get_int64()}; + if (time < 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Mocktime cannot be negative: %s.", time)); + } + SetMockTime(time); + auto node_context = util::AnyPtr<NodeContext>(request.context); + if (node_context) { + for (const auto& chain_client : node_context->chain_clients) { + chain_client->setMockTime(time); + } + } + + return NullUniValue; +}, + }; +} + +#if defined(USE_SYSCALL_SANDBOX) +static RPCHelpMan invokedisallowedsyscall() +{ + return RPCHelpMan{ + "invokedisallowedsyscall", + "\nInvoke a disallowed syscall to trigger a syscall sandbox violation. Used for testing purposes.\n", + {}, + RPCResult{RPCResult::Type::NONE, "", ""}, + RPCExamples{ + HelpExampleCli("invokedisallowedsyscall", "") + HelpExampleRpc("invokedisallowedsyscall", "")}, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + if (!Params().IsTestChain()) { + throw std::runtime_error("invokedisallowedsyscall is used for testing only."); + } + TestDisallowedSandboxCall(); + return NullUniValue; + }, + }; +} +#endif // USE_SYSCALL_SANDBOX + +static RPCHelpMan mockscheduler() +{ + return RPCHelpMan{"mockscheduler", + "\nBump the scheduler into the future (-regtest only)\n", + { + {"delta_time", RPCArg::Type::NUM, RPCArg::Optional::NO, "Number of seconds to forward the scheduler into the future." }, + }, + RPCResult{RPCResult::Type::NONE, "", ""}, + RPCExamples{""}, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + if (!Params().IsMockableChain()) { + throw std::runtime_error("mockscheduler is for regression testing (-regtest mode) only"); + } + + // check params are valid values + RPCTypeCheck(request.params, {UniValue::VNUM}); + int64_t delta_seconds = request.params[0].get_int64(); + if (delta_seconds <= 0 || delta_seconds > 3600) { + throw std::runtime_error("delta_time must be between 1 and 3600 seconds (1 hr)"); + } + + auto node_context = CHECK_NONFATAL(util::AnyPtr<NodeContext>(request.context)); + // protect against null pointer dereference + CHECK_NONFATAL(node_context->scheduler); + node_context->scheduler->MockForward(std::chrono::seconds(delta_seconds)); + + return NullUniValue; +}, + }; +} + +static UniValue RPCLockedMemoryInfo() +{ + LockedPool::Stats stats = LockedPoolManager::Instance().stats(); + UniValue obj(UniValue::VOBJ); + obj.pushKV("used", uint64_t(stats.used)); + obj.pushKV("free", uint64_t(stats.free)); + obj.pushKV("total", uint64_t(stats.total)); + obj.pushKV("locked", uint64_t(stats.locked)); + obj.pushKV("chunks_used", uint64_t(stats.chunks_used)); + obj.pushKV("chunks_free", uint64_t(stats.chunks_free)); + return obj; +} + +#ifdef HAVE_MALLOC_INFO +static std::string RPCMallocInfo() +{ + char *ptr = nullptr; + size_t size = 0; + FILE *f = open_memstream(&ptr, &size); + if (f) { + malloc_info(0, f); + fclose(f); + if (ptr) { + std::string rv(ptr, size); + free(ptr); + return rv; + } + } + return ""; +} +#endif + +static RPCHelpMan getmemoryinfo() +{ + /* Please, avoid using the word "pool" here in the RPC interface or help, + * as users will undoubtedly confuse it with the other "memory pool" + */ + return RPCHelpMan{"getmemoryinfo", + "Returns an object containing information about memory usage.\n", + { + {"mode", RPCArg::Type::STR, RPCArg::Default{"stats"}, "determines what kind of information is returned.\n" + " - \"stats\" returns general statistics about memory usage in the daemon.\n" + " - \"mallocinfo\" returns an XML string describing low-level heap state (only available if compiled with glibc 2.10+)."}, + }, + { + RPCResult{"mode \"stats\"", + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::OBJ, "locked", "Information about locked memory manager", + { + {RPCResult::Type::NUM, "used", "Number of bytes used"}, + {RPCResult::Type::NUM, "free", "Number of bytes available in current arenas"}, + {RPCResult::Type::NUM, "total", "Total number of bytes managed"}, + {RPCResult::Type::NUM, "locked", "Amount of bytes that succeeded locking. If this number is smaller than total, locking pages failed at some point and key data could be swapped to disk."}, + {RPCResult::Type::NUM, "chunks_used", "Number allocated chunks"}, + {RPCResult::Type::NUM, "chunks_free", "Number unused chunks"}, + }}, + } + }, + RPCResult{"mode \"mallocinfo\"", + RPCResult::Type::STR, "", "\"<malloc version=\"1\">...\"" + }, + }, + RPCExamples{ + HelpExampleCli("getmemoryinfo", "") + + HelpExampleRpc("getmemoryinfo", "") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + std::string mode = request.params[0].isNull() ? "stats" : request.params[0].get_str(); + if (mode == "stats") { + UniValue obj(UniValue::VOBJ); + obj.pushKV("locked", RPCLockedMemoryInfo()); + return obj; + } else if (mode == "mallocinfo") { +#ifdef HAVE_MALLOC_INFO + return RPCMallocInfo(); +#else + throw JSONRPCError(RPC_INVALID_PARAMETER, "mallocinfo mode not available"); +#endif + } else { + throw JSONRPCError(RPC_INVALID_PARAMETER, "unknown mode " + mode); + } +}, + }; +} + +static void EnableOrDisableLogCategories(UniValue cats, bool enable) { + cats = cats.get_array(); + for (unsigned int i = 0; i < cats.size(); ++i) { + std::string cat = cats[i].get_str(); + + bool success; + if (enable) { + success = LogInstance().EnableCategory(cat); + } else { + success = LogInstance().DisableCategory(cat); + } + + if (!success) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "unknown logging category " + cat); + } + } +} + +static RPCHelpMan logging() +{ + return RPCHelpMan{"logging", + "Gets and sets the logging configuration.\n" + "When called without an argument, returns the list of categories with status that are currently being debug logged or not.\n" + "When called with arguments, adds or removes categories from debug logging and return the lists above.\n" + "The arguments are evaluated in order \"include\", \"exclude\".\n" + "If an item is both included and excluded, it will thus end up being excluded.\n" + "The valid logging categories are: " + LogInstance().LogCategoriesString() + "\n" + "In addition, the following are available as category names with special meanings:\n" + " - \"all\", \"1\" : represent all logging categories.\n" + " - \"none\", \"0\" : even if other logging categories are specified, ignore all of them.\n" + , + { + {"include", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "The categories to add to debug logging", + { + {"include_category", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "the valid logging category"}, + }}, + {"exclude", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "The categories to remove from debug logging", + { + {"exclude_category", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "the valid logging category"}, + }}, + }, + RPCResult{ + RPCResult::Type::OBJ_DYN, "", "keys are the logging categories, and values indicates its status", + { + {RPCResult::Type::BOOL, "category", "if being debug logged or not. false:inactive, true:active"}, + } + }, + RPCExamples{ + HelpExampleCli("logging", "\"[\\\"all\\\"]\" \"[\\\"http\\\"]\"") + + HelpExampleRpc("logging", "[\"all\"], [\"libevent\"]") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + uint32_t original_log_categories = LogInstance().GetCategoryMask(); + if (request.params[0].isArray()) { + EnableOrDisableLogCategories(request.params[0], true); + } + if (request.params[1].isArray()) { + EnableOrDisableLogCategories(request.params[1], false); + } + uint32_t updated_log_categories = LogInstance().GetCategoryMask(); + uint32_t changed_log_categories = original_log_categories ^ updated_log_categories; + + // Update libevent logging if BCLog::LIBEVENT has changed. + if (changed_log_categories & BCLog::LIBEVENT) { + UpdateHTTPServerLogging(LogInstance().WillLogCategory(BCLog::LIBEVENT)); + } + + UniValue result(UniValue::VOBJ); + for (const auto& logCatActive : LogInstance().LogCategoriesList()) { + result.pushKV(logCatActive.category, logCatActive.active); + } + + return result; +}, + }; +} + +static RPCHelpMan echo(const std::string& name) +{ + return RPCHelpMan{name, + "\nSimply echo back the input arguments. This command is for testing.\n" + "\nIt will return an internal bug report when arg9='trigger_internal_bug' is passed.\n" + "\nThe difference between echo and echojson is that echojson has argument conversion enabled in the client-side table in " + "bitcoin-cli and the GUI. There is no server-side difference.", + { + {"arg0", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, + {"arg1", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, + {"arg2", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, + {"arg3", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, + {"arg4", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, + {"arg5", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, + {"arg6", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, + {"arg7", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, + {"arg8", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, + {"arg9", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, + }, + RPCResult{RPCResult::Type::ANY, "", "Returns whatever was passed in"}, + RPCExamples{""}, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + if (request.params[9].isStr()) { + CHECK_NONFATAL(request.params[9].get_str() != "trigger_internal_bug"); + } + + return request.params; +}, + }; +} + +static RPCHelpMan echo() { return echo("echo"); } +static RPCHelpMan echojson() { return echo("echojson"); } + +static RPCHelpMan echoipc() +{ + return RPCHelpMan{ + "echoipc", + "\nEcho back the input argument, passing it through a spawned process in a multiprocess build.\n" + "This command is for testing.\n", + {{"arg", RPCArg::Type::STR, RPCArg::Optional::NO, "The string to echo",}}, + RPCResult{RPCResult::Type::STR, "echo", "The echoed string."}, + RPCExamples{HelpExampleCli("echo", "\"Hello world\"") + + HelpExampleRpc("echo", "\"Hello world\"")}, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + interfaces::Init& local_init = *EnsureAnyNodeContext(request.context).init; + std::unique_ptr<interfaces::Echo> echo; + if (interfaces::Ipc* ipc = local_init.ipc()) { + // Spawn a new bitcoin-node process and call makeEcho to get a + // client pointer to a interfaces::Echo instance running in + // that process. This is just for testing. A slightly more + // realistic test spawning a different executable instead of + // the same executable would add a new bitcoin-echo executable, + // and spawn bitcoin-echo below instead of bitcoin-node. But + // using bitcoin-node avoids the need to build and install a + // new executable just for this one test. + auto init = ipc->spawnProcess("bitcoin-node"); + echo = init->makeEcho(); + ipc->addCleanup(*echo, [init = init.release()] { delete init; }); + } else { + // IPC support is not available because this is a bitcoind + // process not a bitcoind-node process, so just create a local + // interfaces::Echo object and return it so the `echoipc` RPC + // method will work, and the python test calling `echoipc` + // can expect the same result. + echo = local_init.makeEcho(); + } + return echo->echo(request.params[0].get_str()); + }, + }; +} + +static UniValue SummaryToJSON(const IndexSummary&& summary, std::string index_name) +{ + UniValue ret_summary(UniValue::VOBJ); + if (!index_name.empty() && index_name != summary.name) return ret_summary; + + UniValue entry(UniValue::VOBJ); + entry.pushKV("synced", summary.synced); + entry.pushKV("best_block_height", summary.best_block_height); + ret_summary.pushKV(summary.name, entry); + return ret_summary; +} + +static RPCHelpMan getindexinfo() +{ + return RPCHelpMan{"getindexinfo", + "\nReturns the status of one or all available indices currently running in the node.\n", + { + {"index_name", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "Filter results for an index with a specific name."}, + }, + RPCResult{ + RPCResult::Type::OBJ_DYN, "", "", { + { + RPCResult::Type::OBJ, "name", "The name of the index", + { + {RPCResult::Type::BOOL, "synced", "Whether the index is synced or not"}, + {RPCResult::Type::NUM, "best_block_height", "The block height to which the index is synced"}, + } + }, + }, + }, + RPCExamples{ + HelpExampleCli("getindexinfo", "") + + HelpExampleRpc("getindexinfo", "") + + HelpExampleCli("getindexinfo", "txindex") + + HelpExampleRpc("getindexinfo", "txindex") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + UniValue result(UniValue::VOBJ); + const std::string index_name = request.params[0].isNull() ? "" : request.params[0].get_str(); + + if (g_txindex) { + result.pushKVs(SummaryToJSON(g_txindex->GetSummary(), index_name)); + } + + if (g_coin_stats_index) { + result.pushKVs(SummaryToJSON(g_coin_stats_index->GetSummary(), index_name)); + } + + ForEachBlockFilterIndex([&result, &index_name](const BlockFilterIndex& index) { + result.pushKVs(SummaryToJSON(index.GetSummary(), index_name)); + }); + + return result; +}, + }; +} + +void RegisterNodeRPCCommands(CRPCTable& t) +{ + static const CRPCCommand commands[]{ + {"control", &getmemoryinfo}, + {"control", &logging}, + {"util", &getindexinfo}, + {"hidden", &setmocktime}, + {"hidden", &mockscheduler}, + {"hidden", &echo}, + {"hidden", &echojson}, + {"hidden", &echoipc}, +#if defined(USE_SYSCALL_SANDBOX) + {"hidden", &invokedisallowedsyscall}, +#endif // USE_SYSCALL_SANDBOX + }; + for (const auto& c : commands) { + t.appendCommand(c.name, &c); + } +} diff --git a/src/rpc/output_script.cpp b/src/rpc/output_script.cpp new file mode 100644 index 0000000000..1cf952d0c8 --- /dev/null +++ b/src/rpc/output_script.cpp @@ -0,0 +1,319 @@ +// Copyright (c) 2010 Satoshi Nakamoto +// Copyright (c) 2009-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <key_io.h> +#include <outputtype.h> +#include <pubkey.h> +#include <rpc/protocol.h> +#include <rpc/request.h> +#include <rpc/server.h> +#include <rpc/util.h> +#include <script/descriptor.h> +#include <script/script.h> +#include <script/signingprovider.h> +#include <script/standard.h> +#include <tinyformat.h> +#include <univalue.h> +#include <util/check.h> +#include <util/strencodings.h> + +#include <cstdint> +#include <memory> +#include <optional> +#include <string> +#include <tuple> +#include <vector> + +namespace node { +struct NodeContext; +} +using node::NodeContext; + +static RPCHelpMan validateaddress() +{ + return RPCHelpMan{ + "validateaddress", + "\nReturn information about the given bitcoin address.\n", + { + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address to validate"}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::BOOL, "isvalid", "If the address is valid or not"}, + {RPCResult::Type::STR, "address", /*optional=*/true, "The bitcoin address validated"}, + {RPCResult::Type::STR_HEX, "scriptPubKey", /*optional=*/true, "The hex-encoded scriptPubKey generated by the address"}, + {RPCResult::Type::BOOL, "isscript", /*optional=*/true, "If the key is a script"}, + {RPCResult::Type::BOOL, "iswitness", /*optional=*/true, "If the address is a witness address"}, + {RPCResult::Type::NUM, "witness_version", /*optional=*/true, "The version number of the witness program"}, + {RPCResult::Type::STR_HEX, "witness_program", /*optional=*/true, "The hex value of the witness program"}, + {RPCResult::Type::STR, "error", /*optional=*/true, "Error message, if any"}, + {RPCResult::Type::ARR, "error_locations", /*optional=*/true, "Indices of likely error locations in address, if known (e.g. Bech32 errors)", + { + {RPCResult::Type::NUM, "index", "index of a potential error"}, + }}, + } + }, + RPCExamples{ + HelpExampleCli("validateaddress", "\"" + EXAMPLE_ADDRESS[0] + "\"") + + HelpExampleRpc("validateaddress", "\"" + EXAMPLE_ADDRESS[0] + "\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + std::string error_msg; + std::vector<int> error_locations; + CTxDestination dest = DecodeDestination(request.params[0].get_str(), error_msg, &error_locations); + const bool isValid = IsValidDestination(dest); + CHECK_NONFATAL(isValid == error_msg.empty()); + + UniValue ret(UniValue::VOBJ); + ret.pushKV("isvalid", isValid); + if (isValid) { + std::string currentAddress = EncodeDestination(dest); + ret.pushKV("address", currentAddress); + + CScript scriptPubKey = GetScriptForDestination(dest); + ret.pushKV("scriptPubKey", HexStr(scriptPubKey)); + + UniValue detail = DescribeAddress(dest); + ret.pushKVs(detail); + } else { + UniValue error_indices(UniValue::VARR); + for (int i : error_locations) error_indices.push_back(i); + ret.pushKV("error_locations", error_indices); + ret.pushKV("error", error_msg); + } + + return ret; + }, + }; +} + +static RPCHelpMan createmultisig() +{ + return RPCHelpMan{"createmultisig", + "\nCreates a multi-signature address with n signature of m keys required.\n" + "It returns a json object with the address and redeemScript.\n", + { + {"nrequired", RPCArg::Type::NUM, RPCArg::Optional::NO, "The number of required signatures out of the n keys."}, + {"keys", RPCArg::Type::ARR, RPCArg::Optional::NO, "The hex-encoded public keys.", + { + {"key", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "The hex-encoded public key"}, + }}, + {"address_type", RPCArg::Type::STR, RPCArg::Default{"legacy"}, "The address type to use. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR, "address", "The value of the new multisig address."}, + {RPCResult::Type::STR_HEX, "redeemScript", "The string value of the hex-encoded redemption script."}, + {RPCResult::Type::STR, "descriptor", "The descriptor for this multisig"}, + {RPCResult::Type::ARR, "warnings", /*optional=*/true, "Any warnings resulting from the creation of this multisig", + { + {RPCResult::Type::STR, "", ""}, + }}, + } + }, + RPCExamples{ + "\nCreate a multisig address from 2 public keys\n" + + HelpExampleCli("createmultisig", "2 \"[\\\"03789ed0bb717d88f7d321a368d905e7430207ebbd82bd342cf11ae157a7ace5fd\\\",\\\"03dbc6764b8884a92e871274b87583e6d5c2a58819473e17e107ef3f6aa5a61626\\\"]\"") + + "\nAs a JSON-RPC call\n" + + HelpExampleRpc("createmultisig", "2, [\"03789ed0bb717d88f7d321a368d905e7430207ebbd82bd342cf11ae157a7ace5fd\",\"03dbc6764b8884a92e871274b87583e6d5c2a58819473e17e107ef3f6aa5a61626\"]") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + int required = request.params[0].get_int(); + + // Get the public keys + const UniValue& keys = request.params[1].get_array(); + std::vector<CPubKey> pubkeys; + for (unsigned int i = 0; i < keys.size(); ++i) { + if (IsHex(keys[i].get_str()) && (keys[i].get_str().length() == 66 || keys[i].get_str().length() == 130)) { + pubkeys.push_back(HexToPubKey(keys[i].get_str())); + } else { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Invalid public key: %s\n.", keys[i].get_str())); + } + } + + // Get the output type + OutputType output_type = OutputType::LEGACY; + if (!request.params[2].isNull()) { + std::optional<OutputType> parsed = ParseOutputType(request.params[2].get_str()); + if (!parsed) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Unknown address type '%s'", request.params[2].get_str())); + } else if (parsed.value() == OutputType::BECH32M) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "createmultisig cannot create bech32m multisig addresses"); + } + output_type = parsed.value(); + } + + // Construct using pay-to-script-hash: + FillableSigningProvider keystore; + CScript inner; + const CTxDestination dest = AddAndGetMultisigDestination(required, pubkeys, output_type, keystore, inner); + + // Make the descriptor + std::unique_ptr<Descriptor> descriptor = InferDescriptor(GetScriptForDestination(dest), keystore); + + UniValue result(UniValue::VOBJ); + result.pushKV("address", EncodeDestination(dest)); + result.pushKV("redeemScript", HexStr(inner)); + result.pushKV("descriptor", descriptor->ToString()); + + UniValue warnings(UniValue::VARR); + if (!request.params[2].isNull() && OutputTypeFromDestination(dest) != output_type) { + // Only warns if the user has explicitly chosen an address type we cannot generate + warnings.push_back("Unable to make chosen address type, please ensure no uncompressed public keys are present."); + } + if (warnings.size()) result.pushKV("warnings", warnings); + + return result; + }, + }; +} + +static RPCHelpMan getdescriptorinfo() +{ + const std::string EXAMPLE_DESCRIPTOR = "wpkh([d34db33f/84h/0h/0h]0279be667ef9dcbbac55a06295Ce870b07029Bfcdb2dce28d959f2815b16f81798)"; + + return RPCHelpMan{"getdescriptorinfo", + {"\nAnalyses a descriptor.\n"}, + { + {"descriptor", RPCArg::Type::STR, RPCArg::Optional::NO, "The descriptor."}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR, "descriptor", "The descriptor in canonical form, without private keys"}, + {RPCResult::Type::STR, "checksum", "The checksum for the input descriptor"}, + {RPCResult::Type::BOOL, "isrange", "Whether the descriptor is ranged"}, + {RPCResult::Type::BOOL, "issolvable", "Whether the descriptor is solvable"}, + {RPCResult::Type::BOOL, "hasprivatekeys", "Whether the input descriptor contained at least one private key"}, + } + }, + RPCExamples{ + "Analyse a descriptor\n" + + HelpExampleCli("getdescriptorinfo", "\"" + EXAMPLE_DESCRIPTOR + "\"") + + HelpExampleRpc("getdescriptorinfo", "\"" + EXAMPLE_DESCRIPTOR + "\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + RPCTypeCheck(request.params, {UniValue::VSTR}); + + FlatSigningProvider provider; + std::string error; + auto desc = Parse(request.params[0].get_str(), provider, error); + if (!desc) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, error); + } + + UniValue result(UniValue::VOBJ); + result.pushKV("descriptor", desc->ToString()); + result.pushKV("checksum", GetDescriptorChecksum(request.params[0].get_str())); + result.pushKV("isrange", desc->IsRange()); + result.pushKV("issolvable", desc->IsSolvable()); + result.pushKV("hasprivatekeys", provider.keys.size() > 0); + return result; + }, + }; +} + +static RPCHelpMan deriveaddresses() +{ + const std::string EXAMPLE_DESCRIPTOR = "wpkh([d34db33f/84h/0h/0h]xpub6DJ2dNUysrn5Vt36jH2KLBT2i1auw1tTSSomg8PhqNiUtx8QX2SvC9nrHu81fT41fvDUnhMjEzQgXnQjKEu3oaqMSzhSrHMxyyoEAmUHQbY/0/*)#cjjspncu"; + + return RPCHelpMan{"deriveaddresses", + {"\nDerives one or more addresses corresponding to an output descriptor.\n" + "Examples of output descriptors are:\n" + " pkh(<pubkey>) P2PKH outputs for the given pubkey\n" + " wpkh(<pubkey>) Native segwit P2PKH outputs for the given pubkey\n" + " sh(multi(<n>,<pubkey>,<pubkey>,...)) P2SH-multisig outputs for the given threshold and pubkeys\n" + " raw(<hex script>) Outputs whose scriptPubKey equals the specified hex scripts\n" + "\nIn the above, <pubkey> either refers to a fixed public key in hexadecimal notation, or to an xpub/xprv optionally followed by one\n" + "or more path elements separated by \"/\", where \"h\" represents a hardened child key.\n" + "For more information on output descriptors, see the documentation in the doc/descriptors.md file.\n"}, + { + {"descriptor", RPCArg::Type::STR, RPCArg::Optional::NO, "The descriptor."}, + {"range", RPCArg::Type::RANGE, RPCArg::Optional::OMITTED_NAMED_ARG, "If a ranged descriptor is used, this specifies the end or the range (in [begin,end] notation) to derive."}, + }, + RPCResult{ + RPCResult::Type::ARR, "", "", + { + {RPCResult::Type::STR, "address", "the derived addresses"}, + } + }, + RPCExamples{ + "First three native segwit receive addresses\n" + + HelpExampleCli("deriveaddresses", "\"" + EXAMPLE_DESCRIPTOR + "\" \"[0,2]\"") + + HelpExampleRpc("deriveaddresses", "\"" + EXAMPLE_DESCRIPTOR + "\", \"[0,2]\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + RPCTypeCheck(request.params, {UniValue::VSTR, UniValueType()}); // Range argument is checked later + const std::string desc_str = request.params[0].get_str(); + + int64_t range_begin = 0; + int64_t range_end = 0; + + if (request.params.size() >= 2 && !request.params[1].isNull()) { + std::tie(range_begin, range_end) = ParseDescriptorRange(request.params[1]); + } + + FlatSigningProvider key_provider; + std::string error; + auto desc = Parse(desc_str, key_provider, error, /* require_checksum = */ true); + if (!desc) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, error); + } + + if (!desc->IsRange() && request.params.size() > 1) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Range should not be specified for an un-ranged descriptor"); + } + + if (desc->IsRange() && request.params.size() == 1) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Range must be specified for a ranged descriptor"); + } + + UniValue addresses(UniValue::VARR); + + for (int i = range_begin; i <= range_end; ++i) { + FlatSigningProvider provider; + std::vector<CScript> scripts; + if (!desc->Expand(i, key_provider, scripts, provider)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Cannot derive script without private keys"); + } + + for (const CScript& script : scripts) { + CTxDestination dest; + if (!ExtractDestination(script, dest)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Descriptor does not have a corresponding address"); + } + + addresses.push_back(EncodeDestination(dest)); + } + } + + // This should not be possible, but an assert seems overkill: + if (addresses.empty()) { + throw JSONRPCError(RPC_MISC_ERROR, "Unexpected empty result"); + } + + return addresses; + }, + }; +} + +void RegisterOutputScriptRPCCommands(CRPCTable& t) +{ + static const CRPCCommand commands[]{ + {"util", &validateaddress}, + {"util", &createmultisig}, + {"util", &deriveaddresses}, + {"util", &getdescriptorinfo}, + }; + for (const auto& c : commands) { + t.appendCommand(c.name, &c); + } +} diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 6272a7c8cf..e8713fbd2e 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -11,7 +11,6 @@ #include <core_io.h> #include <index/txindex.h> #include <key_io.h> -#include <merkleblock.h> #include <node/blockstorage.h> #include <node/coin.h> #include <node/context.h> @@ -34,9 +33,10 @@ #include <script/standard.h> #include <uint256.h> #include <util/bip32.h> -#include <util/moneystr.h> +#include <util/check.h> #include <util/strencodings.h> #include <util/string.h> +#include <util/vector.h> #include <validation.h> #include <validationinterface.h> @@ -47,7 +47,6 @@ using node::AnalyzePSBT; using node::BroadcastTransaction; -using node::DEFAULT_MAX_RAW_TX_FEE_RATE; using node::FindCoins; using node::GetTransaction; using node::NodeContext; @@ -61,13 +60,13 @@ static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& // Blockchain contextual information (confirmations and blocktime) is not // available to code in bitcoin-common, so we query them here and push the // data into the returned UniValue. - TxToUniv(tx, uint256(), entry, true, RPCSerializationFlags()); + TxToUniv(tx, /*block_hash=*/uint256(), entry, /*include_hex=*/true, RPCSerializationFlags()); if (!hashBlock.IsNull()) { LOCK(cs_main); entry.pushKV("blockhash", hashBlock.GetHex()); - CBlockIndex* pindex = active_chainstate.m_blockman.LookupBlockIndex(hashBlock); + const CBlockIndex* pindex = active_chainstate.m_blockman.LookupBlockIndex(hashBlock); if (pindex) { if (active_chainstate.m_chain.Contains(pindex)) { entry.pushKV("confirmations", 1 + active_chainstate.m_chain.Height() - pindex->nHeight); @@ -80,6 +79,54 @@ static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& } } +static std::vector<RPCResult> DecodeTxDoc(const std::string& txid_field_doc) +{ + return { + {RPCResult::Type::STR_HEX, "txid", txid_field_doc}, + {RPCResult::Type::STR_HEX, "hash", "The transaction hash (differs from txid for witness transactions)"}, + {RPCResult::Type::NUM, "size", "The serialized transaction size"}, + {RPCResult::Type::NUM, "vsize", "The virtual transaction size (differs from size for witness transactions)"}, + {RPCResult::Type::NUM, "weight", "The transaction's weight (between vsize*4-3 and vsize*4)"}, + {RPCResult::Type::NUM, "version", "The version"}, + {RPCResult::Type::NUM_TIME, "locktime", "The lock time"}, + {RPCResult::Type::ARR, "vin", "", + { + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR_HEX, "coinbase", /*optional=*/true, "The coinbase value (only if coinbase transaction)"}, + {RPCResult::Type::STR_HEX, "txid", /*optional=*/true, "The transaction id (if not coinbase transaction)"}, + {RPCResult::Type::NUM, "vout", /*optional=*/true, "The output number (if not coinbase transaction)"}, + {RPCResult::Type::OBJ, "scriptSig", /*optional=*/true, "The script (if not coinbase transaction)", + { + {RPCResult::Type::STR, "asm", "asm"}, + {RPCResult::Type::STR_HEX, "hex", "hex"}, + }}, + {RPCResult::Type::ARR, "txinwitness", /*optional=*/true, "", + { + {RPCResult::Type::STR_HEX, "hex", "hex-encoded witness data (if any)"}, + }}, + {RPCResult::Type::NUM, "sequence", "The script sequence number"}, + }}, + }}, + {RPCResult::Type::ARR, "vout", "", + { + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR_AMOUNT, "value", "The value in " + CURRENCY_UNIT}, + {RPCResult::Type::NUM, "n", "index"}, + {RPCResult::Type::OBJ, "scriptPubKey", "", + { + {RPCResult::Type::STR, "asm", "the asm"}, + {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, + {RPCResult::Type::STR_HEX, "hex", "the hex"}, + {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, + {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, + }}, + }}, + }}, + }; +} + static std::vector<RPCArg> CreateTxDoc() { return { @@ -121,7 +168,7 @@ static RPCHelpMan getrawtransaction() { return RPCHelpMan{ "getrawtransaction", - "\nReturn the raw transaction data.\n" + "Return the raw transaction data.\n" "\nBy default, this call only returns a transaction if it is in the mempool. If -txindex is enabled\n" "and no blockhash argument is passed, it will return the transaction if it is in the mempool or any block.\n" @@ -130,7 +177,7 @@ static RPCHelpMan getrawtransaction() "\nHint: Use gettransaction for wallet transactions.\n" "\nIf verbose is 'true', returns an Object with information about 'txid'.\n" - "If verbose is 'false' or omitted, returns a string that is serialized, hex-encoded data for 'txid'.\n", + "If verbose is 'false' or omitted, returns a string that is serialized, hex-encoded data for 'txid'.", { {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id"}, {"verbose", RPCArg::Type::BOOL, RPCArg::Default{false}, "If false, return a string, otherwise return a json object"}, @@ -142,55 +189,16 @@ static RPCHelpMan getrawtransaction() }, RPCResult{"if verbose is set to true", RPCResult::Type::OBJ, "", "", + Cat<std::vector<RPCResult>>( { {RPCResult::Type::BOOL, "in_active_chain", /*optional=*/true, "Whether specified block is in the active chain or not (only present with explicit \"blockhash\" argument)"}, - {RPCResult::Type::STR_HEX, "hex", "The serialized, hex-encoded data for 'txid'"}, - {RPCResult::Type::STR_HEX, "txid", "The transaction id (same as provided)"}, - {RPCResult::Type::STR_HEX, "hash", "The transaction hash (differs from txid for witness transactions)"}, - {RPCResult::Type::NUM, "size", "The serialized transaction size"}, - {RPCResult::Type::NUM, "vsize", "The virtual transaction size (differs from size for witness transactions)"}, - {RPCResult::Type::NUM, "weight", "The transaction's weight (between vsize*4-3 and vsize*4)"}, - {RPCResult::Type::NUM, "version", "The version"}, - {RPCResult::Type::NUM_TIME, "locktime", "The lock time"}, - {RPCResult::Type::ARR, "vin", "", - { - {RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR_HEX, "txid", "The transaction id"}, - {RPCResult::Type::NUM, "vout", "The output number"}, - {RPCResult::Type::OBJ, "scriptSig", "The script", - { - {RPCResult::Type::STR, "asm", "asm"}, - {RPCResult::Type::STR_HEX, "hex", "hex"}, - }}, - {RPCResult::Type::NUM, "sequence", "The script sequence number"}, - {RPCResult::Type::ARR, "txinwitness", /*optional=*/true, "", - { - {RPCResult::Type::STR_HEX, "hex", "hex-encoded witness data (if any)"}, - }}, - }}, - }}, - {RPCResult::Type::ARR, "vout", "", - { - {RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::NUM, "value", "The value in " + CURRENCY_UNIT}, - {RPCResult::Type::NUM, "n", "index"}, - {RPCResult::Type::OBJ, "scriptPubKey", "", - { - {RPCResult::Type::STR, "asm", "the asm"}, - {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, - {RPCResult::Type::STR, "hex", "the hex"}, - {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, - {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, - }}, - }}, - }}, {RPCResult::Type::STR_HEX, "blockhash", /*optional=*/true, "the block hash"}, {RPCResult::Type::NUM, "confirmations", /*optional=*/true, "The confirmations"}, {RPCResult::Type::NUM_TIME, "blocktime", /*optional=*/true, "The block time expressed in " + UNIX_EPOCH_TIME}, {RPCResult::Type::NUM, "time", /*optional=*/true, "Same as \"blocktime\""}, - } + {RPCResult::Type::STR_HEX, "hex", "The serialized, hex-encoded data for 'txid'"}, + }, + DecodeTxDoc(/*txid_field_doc=*/"The transaction id (same as provided)")), }, }, RPCExamples{ @@ -207,7 +215,7 @@ static RPCHelpMan getrawtransaction() bool in_active_chain = true; uint256 hash = ParseHashV(request.params[0], "parameter 1"); - CBlockIndex* blockindex = nullptr; + const CBlockIndex* blockindex = nullptr; if (hash == Params().GenesisBlock().hashMerkleRoot) { // Special exception for the genesis block coinbase transaction @@ -268,155 +276,6 @@ static RPCHelpMan getrawtransaction() }; } -static RPCHelpMan gettxoutproof() -{ - return RPCHelpMan{"gettxoutproof", - "\nReturns a hex-encoded proof that \"txid\" was included in a block.\n" - "\nNOTE: By default this function only works sometimes. This is when there is an\n" - "unspent output in the utxo for this transaction. To make it always work,\n" - "you need to maintain a transaction index, using the -txindex command line option or\n" - "specify the block in which the transaction is included manually (by blockhash).\n", - { - {"txids", RPCArg::Type::ARR, RPCArg::Optional::NO, "The txids to filter", - { - {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A transaction hash"}, - }, - }, - {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED_NAMED_ARG, "If specified, looks for txid in the block with this hash"}, - }, - RPCResult{ - RPCResult::Type::STR, "data", "A string that is a serialized, hex-encoded data for the proof." - }, - RPCExamples{""}, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - std::set<uint256> setTxids; - UniValue txids = request.params[0].get_array(); - if (txids.empty()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter 'txids' cannot be empty"); - } - for (unsigned int idx = 0; idx < txids.size(); idx++) { - auto ret = setTxids.insert(ParseHashV(txids[idx], "txid")); - if (!ret.second) { - throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid parameter, duplicated txid: ") + txids[idx].get_str()); - } - } - - CBlockIndex* pblockindex = nullptr; - uint256 hashBlock; - ChainstateManager& chainman = EnsureAnyChainman(request.context); - if (!request.params[1].isNull()) { - LOCK(cs_main); - hashBlock = ParseHashV(request.params[1], "blockhash"); - pblockindex = chainman.m_blockman.LookupBlockIndex(hashBlock); - if (!pblockindex) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); - } - } else { - LOCK(cs_main); - CChainState& active_chainstate = chainman.ActiveChainstate(); - - // Loop through txids and try to find which block they're in. Exit loop once a block is found. - for (const auto& tx : setTxids) { - const Coin& coin = AccessByTxid(active_chainstate.CoinsTip(), tx); - if (!coin.IsSpent()) { - pblockindex = active_chainstate.m_chain[coin.nHeight]; - break; - } - } - } - - - // Allow txindex to catch up if we need to query it and before we acquire cs_main. - if (g_txindex && !pblockindex) { - g_txindex->BlockUntilSyncedToCurrentChain(); - } - - LOCK(cs_main); - - if (pblockindex == nullptr) { - const CTransactionRef tx = GetTransaction(/* block_index */ nullptr, /* mempool */ nullptr, *setTxids.begin(), Params().GetConsensus(), hashBlock); - if (!tx || hashBlock.IsNull()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not yet in block"); - } - pblockindex = chainman.m_blockman.LookupBlockIndex(hashBlock); - if (!pblockindex) { - throw JSONRPCError(RPC_INTERNAL_ERROR, "Transaction index corrupt"); - } - } - - CBlock block; - if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) { - throw JSONRPCError(RPC_INTERNAL_ERROR, "Can't read block from disk"); - } - - unsigned int ntxFound = 0; - for (const auto& tx : block.vtx) { - if (setTxids.count(tx->GetHash())) { - ntxFound++; - } - } - if (ntxFound != setTxids.size()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Not all transactions found in specified or retrieved block"); - } - - CDataStream ssMB(SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS); - CMerkleBlock mb(block, setTxids); - ssMB << mb; - std::string strHex = HexStr(ssMB); - return strHex; -}, - }; -} - -static RPCHelpMan verifytxoutproof() -{ - return RPCHelpMan{"verifytxoutproof", - "\nVerifies that a proof points to a transaction in a block, returning the transaction it commits to\n" - "and throwing an RPC error if the block is not in our best chain\n", - { - {"proof", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex-encoded proof generated by gettxoutproof"}, - }, - RPCResult{ - RPCResult::Type::ARR, "", "", - { - {RPCResult::Type::STR_HEX, "txid", "The txid(s) which the proof commits to, or empty array if the proof cannot be validated."}, - } - }, - RPCExamples{""}, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - CDataStream ssMB(ParseHexV(request.params[0], "proof"), SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS); - CMerkleBlock merkleBlock; - ssMB >> merkleBlock; - - UniValue res(UniValue::VARR); - - std::vector<uint256> vMatch; - std::vector<unsigned int> vIndex; - if (merkleBlock.txn.ExtractMatches(vMatch, vIndex) != merkleBlock.header.hashMerkleRoot) - return res; - - ChainstateManager& chainman = EnsureAnyChainman(request.context); - LOCK(cs_main); - - const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(merkleBlock.header.GetHash()); - if (!pindex || !chainman.ActiveChain().Contains(pindex) || pindex->nTx == 0) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found in chain"); - } - - // Check if proof is valid, only add results if so - if (pindex->nTx == merkleBlock.txn.GetNumTransactions()) { - for (const uint256& hash : vMatch) { - res.push_back(hash.GetHex()); - } - } - - return res; -}, - }; -} - static RPCHelpMan createrawtransaction() { return RPCHelpMan{"createrawtransaction", @@ -459,7 +318,7 @@ static RPCHelpMan createrawtransaction() static RPCHelpMan decoderawtransaction() { return RPCHelpMan{"decoderawtransaction", - "\nReturn a JSON object representing the serialized, hex-encoded transaction.\n", + "Return a JSON object representing the serialized, hex-encoded transaction.", { {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction hex string"}, {"iswitness", RPCArg::Type::BOOL, RPCArg::DefaultHint{"depends on heuristic tests"}, "Whether the transaction hex is a serialized witness transaction.\n" @@ -472,50 +331,7 @@ static RPCHelpMan decoderawtransaction() }, RPCResult{ RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR_HEX, "txid", "The transaction id"}, - {RPCResult::Type::STR_HEX, "hash", "The transaction hash (differs from txid for witness transactions)"}, - {RPCResult::Type::NUM, "size", "The transaction size"}, - {RPCResult::Type::NUM, "vsize", "The virtual transaction size (differs from size for witness transactions)"}, - {RPCResult::Type::NUM, "weight", "The transaction's weight (between vsize*4 - 3 and vsize*4)"}, - {RPCResult::Type::NUM, "version", "The version"}, - {RPCResult::Type::NUM_TIME, "locktime", "The lock time"}, - {RPCResult::Type::ARR, "vin", "", - { - {RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR_HEX, "coinbase", /*optional=*/true, ""}, - {RPCResult::Type::STR_HEX, "txid", /*optional=*/true, "The transaction id"}, - {RPCResult::Type::NUM, "vout", /*optional=*/true, "The output number"}, - {RPCResult::Type::OBJ, "scriptSig", /*optional=*/true, "The script", - { - {RPCResult::Type::STR, "asm", "asm"}, - {RPCResult::Type::STR_HEX, "hex", "hex"}, - }}, - {RPCResult::Type::ARR, "txinwitness", /*optional=*/true, "", - { - {RPCResult::Type::STR_HEX, "hex", "hex-encoded witness data (if any)"}, - }}, - {RPCResult::Type::NUM, "sequence", "The script sequence number"}, - }}, - }}, - {RPCResult::Type::ARR, "vout", "", - { - {RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::NUM, "value", "The value in " + CURRENCY_UNIT}, - {RPCResult::Type::NUM, "n", "index"}, - {RPCResult::Type::OBJ, "scriptPubKey", "", - { - {RPCResult::Type::STR, "asm", "the asm"}, - {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, - {RPCResult::Type::STR_HEX, "hex", "the hex"}, - {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, - {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, - }}, - }}, - }}, - } + DecodeTxDoc(/*txid_field_doc=*/"The transaction id"), }, RPCExamples{ HelpExampleCli("decoderawtransaction", "\"hexstring\"") @@ -535,7 +351,7 @@ static RPCHelpMan decoderawtransaction() } UniValue result(UniValue::VOBJ); - TxToUniv(CTransaction(std::move(mtx)), uint256(), result, false); + TxToUniv(CTransaction(std::move(mtx)), /*block_hash=*/uint256(), /*entry=*/result, /*include_hex=*/false); return result; }, @@ -587,7 +403,7 @@ static RPCHelpMan decodescript() } else { // Empty scripts are valid } - ScriptPubKeyToUniv(script, r, /* include_hex */ false); + ScriptToUniv(script, /*out=*/r, /*include_hex=*/false, /*include_address=*/true); std::vector<std::vector<unsigned char>> solutions_data; const TxoutType which_type{Solver(script, solutions_data)}; @@ -651,7 +467,7 @@ static RPCHelpMan decodescript() // Should not be wrapped return false; } // no default case, so the compiler can warn about missing cases - CHECK_NONFATAL(false); + NONFATAL_UNREACHABLE(); }()}; if (can_wrap_P2WSH) { UniValue sr(UniValue::VOBJ); @@ -664,7 +480,7 @@ static RPCHelpMan decodescript() // Scripts that are not fit for P2WPKH are encoded as P2WSH. segwitScr = GetScriptForDestination(WitnessV0ScriptHash(script)); } - ScriptPubKeyToUniv(segwitScr, sr, /* include_hex */ true); + ScriptToUniv(segwitScr, /*out=*/sr, /*include_hex=*/true, /*include_address=*/true); sr.pushKV("p2sh-segwit", EncodeDestination(ScriptHash(segwitScr))); r.pushKV("segwit", sr); } @@ -751,7 +567,7 @@ static RPCHelpMan combinerawtransaction() sigdata.MergeSignatureData(DataFromTransaction(txv, i, coin.out)); } } - ProduceSignature(DUMMY_SIGNING_PROVIDER, MutableTransactionSignatureCreator(&mergedTx, i, coin.out.nValue, 1), coin.out.scriptPubKey, sigdata); + ProduceSignature(DUMMY_SIGNING_PROVIDER, MutableTransactionSignatureCreator(mergedTx, i, coin.out.nValue, 1), coin.out.scriptPubKey, sigdata); UpdateInput(txin, sigdata); } @@ -864,210 +680,6 @@ static RPCHelpMan signrawtransactionwithkey() }; } -static RPCHelpMan sendrawtransaction() -{ - return RPCHelpMan{"sendrawtransaction", - "\nSubmit a raw transaction (serialized, hex-encoded) to local node and network.\n" - "\nThe transaction will be sent unconditionally to all peers, so using sendrawtransaction\n" - "for manual rebroadcast may degrade privacy by leaking the transaction's origin, as\n" - "nodes will normally not rebroadcast non-wallet transactions already in their mempool.\n" - "\nA specific exception, RPC_TRANSACTION_ALREADY_IN_CHAIN, may throw if the transaction cannot be added to the mempool.\n" - "\nRelated RPCs: createrawtransaction, signrawtransactionwithkey\n", - { - {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex string of the raw transaction"}, - {"maxfeerate", RPCArg::Type::AMOUNT, RPCArg::Default{FormatMoney(DEFAULT_MAX_RAW_TX_FEE_RATE.GetFeePerK())}, - "Reject transactions whose fee rate is higher than the specified value, expressed in " + CURRENCY_UNIT + - "/kvB.\nSet to 0 to accept any fee rate.\n"}, - }, - RPCResult{ - RPCResult::Type::STR_HEX, "", "The transaction hash in hex" - }, - RPCExamples{ - "\nCreate a transaction\n" - + HelpExampleCli("createrawtransaction", "\"[{\\\"txid\\\" : \\\"mytxid\\\",\\\"vout\\\":0}]\" \"{\\\"myaddress\\\":0.01}\"") + - "Sign the transaction, and get back the hex\n" - + HelpExampleCli("signrawtransactionwithwallet", "\"myhex\"") + - "\nSend the transaction (signed hex)\n" - + HelpExampleCli("sendrawtransaction", "\"signedhex\"") + - "\nAs a JSON-RPC call\n" - + HelpExampleRpc("sendrawtransaction", "\"signedhex\"") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - RPCTypeCheck(request.params, { - UniValue::VSTR, - UniValueType(), // VNUM or VSTR, checked inside AmountFromValue() - }); - - CMutableTransaction mtx; - if (!DecodeHexTx(mtx, request.params[0].get_str())) { - throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed. Make sure the tx has at least one input."); - } - CTransactionRef tx(MakeTransactionRef(std::move(mtx))); - - const CFeeRate max_raw_tx_fee_rate = request.params[1].isNull() ? - DEFAULT_MAX_RAW_TX_FEE_RATE : - CFeeRate(AmountFromValue(request.params[1])); - - int64_t virtual_size = GetVirtualTransactionSize(*tx); - CAmount max_raw_tx_fee = max_raw_tx_fee_rate.GetFee(virtual_size); - - std::string err_string; - AssertLockNotHeld(cs_main); - NodeContext& node = EnsureAnyNodeContext(request.context); - const TransactionError err = BroadcastTransaction(node, tx, err_string, max_raw_tx_fee, /*relay*/ true, /*wait_callback*/ true); - if (TransactionError::OK != err) { - throw JSONRPCTransactionError(err, err_string); - } - - return tx->GetHash().GetHex(); -}, - }; -} - -static RPCHelpMan testmempoolaccept() -{ - return RPCHelpMan{"testmempoolaccept", - "\nReturns result of mempool acceptance tests indicating if raw transaction(s) (serialized, hex-encoded) would be accepted by mempool.\n" - "\nIf multiple transactions are passed in, parents must come before children and package policies apply: the transactions cannot conflict with any mempool transactions or each other.\n" - "\nIf one transaction fails, other transactions may not be fully validated (the 'allowed' key will be blank).\n" - "\nThe maximum number of transactions allowed is " + ToString(MAX_PACKAGE_COUNT) + ".\n" - "\nThis checks if transactions violate the consensus or policy rules.\n" - "\nSee sendrawtransaction call.\n", - { - {"rawtxs", RPCArg::Type::ARR, RPCArg::Optional::NO, "An array of hex strings of raw transactions.", - { - {"rawtx", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, - }, - }, - {"maxfeerate", RPCArg::Type::AMOUNT, RPCArg::Default{FormatMoney(DEFAULT_MAX_RAW_TX_FEE_RATE.GetFeePerK())}, - "Reject transactions whose fee rate is higher than the specified value, expressed in " + CURRENCY_UNIT + "/kvB\n"}, - }, - RPCResult{ - RPCResult::Type::ARR, "", "The result of the mempool acceptance test for each raw transaction in the input array.\n" - "Returns results for each transaction in the same order they were passed in.\n" - "Transactions that cannot be fully validated due to failures in other transactions will not contain an 'allowed' result.\n", - { - {RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR_HEX, "txid", "The transaction hash in hex"}, - {RPCResult::Type::STR_HEX, "wtxid", "The transaction witness hash in hex"}, - {RPCResult::Type::STR, "package-error", /*optional=*/true, "Package validation error, if any (only possible if rawtxs had more than 1 transaction)."}, - {RPCResult::Type::BOOL, "allowed", /*optional=*/true, "Whether this tx would be accepted to the mempool and pass client-specified maxfeerate. " - "If not present, the tx was not fully validated due to a failure in another tx in the list."}, - {RPCResult::Type::NUM, "vsize", /*optional=*/true, "Virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted (only present when 'allowed' is true)"}, - {RPCResult::Type::OBJ, "fees", /*optional=*/true, "Transaction fees (only present if 'allowed' is true)", - { - {RPCResult::Type::STR_AMOUNT, "base", "transaction fee in " + CURRENCY_UNIT}, - }}, - {RPCResult::Type::STR, "reject-reason", /*optional=*/true, "Rejection string (only present when 'allowed' is false)"}, - }}, - } - }, - RPCExamples{ - "\nCreate a transaction\n" - + HelpExampleCli("createrawtransaction", "\"[{\\\"txid\\\" : \\\"mytxid\\\",\\\"vout\\\":0}]\" \"{\\\"myaddress\\\":0.01}\"") + - "Sign the transaction, and get back the hex\n" - + HelpExampleCli("signrawtransactionwithwallet", "\"myhex\"") + - "\nTest acceptance of the transaction (signed hex)\n" - + HelpExampleCli("testmempoolaccept", R"('["signedhex"]')") + - "\nAs a JSON-RPC call\n" - + HelpExampleRpc("testmempoolaccept", "[\"signedhex\"]") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - RPCTypeCheck(request.params, { - UniValue::VARR, - UniValueType(), // VNUM or VSTR, checked inside AmountFromValue() - }); - const UniValue raw_transactions = request.params[0].get_array(); - if (raw_transactions.size() < 1 || raw_transactions.size() > MAX_PACKAGE_COUNT) { - throw JSONRPCError(RPC_INVALID_PARAMETER, - "Array must contain between 1 and " + ToString(MAX_PACKAGE_COUNT) + " transactions."); - } - - const CFeeRate max_raw_tx_fee_rate = request.params[1].isNull() ? - DEFAULT_MAX_RAW_TX_FEE_RATE : - CFeeRate(AmountFromValue(request.params[1])); - - std::vector<CTransactionRef> txns; - txns.reserve(raw_transactions.size()); - for (const auto& rawtx : raw_transactions.getValues()) { - CMutableTransaction mtx; - if (!DecodeHexTx(mtx, rawtx.get_str())) { - throw JSONRPCError(RPC_DESERIALIZATION_ERROR, - "TX decode failed: " + rawtx.get_str() + " Make sure the tx has at least one input."); - } - txns.emplace_back(MakeTransactionRef(std::move(mtx))); - } - - NodeContext& node = EnsureAnyNodeContext(request.context); - CTxMemPool& mempool = EnsureMemPool(node); - ChainstateManager& chainman = EnsureChainman(node); - CChainState& chainstate = chainman.ActiveChainstate(); - const PackageMempoolAcceptResult package_result = [&] { - LOCK(::cs_main); - if (txns.size() > 1) return ProcessNewPackage(chainstate, mempool, txns, /* test_accept */ true); - return PackageMempoolAcceptResult(txns[0]->GetWitnessHash(), - chainman.ProcessTransaction(txns[0], /*test_accept=*/ true)); - }(); - - UniValue rpc_result(UniValue::VARR); - // We will check transaction fees while we iterate through txns in order. If any transaction fee - // exceeds maxfeerate, we will leave the rest of the validation results blank, because it - // doesn't make sense to return a validation result for a transaction if its ancestor(s) would - // not be submitted. - bool exit_early{false}; - for (const auto& tx : txns) { - UniValue result_inner(UniValue::VOBJ); - result_inner.pushKV("txid", tx->GetHash().GetHex()); - result_inner.pushKV("wtxid", tx->GetWitnessHash().GetHex()); - if (package_result.m_state.GetResult() == PackageValidationResult::PCKG_POLICY) { - result_inner.pushKV("package-error", package_result.m_state.GetRejectReason()); - } - auto it = package_result.m_tx_results.find(tx->GetWitnessHash()); - if (exit_early || it == package_result.m_tx_results.end()) { - // Validation unfinished. Just return the txid and wtxid. - rpc_result.push_back(result_inner); - continue; - } - const auto& tx_result = it->second; - // Package testmempoolaccept doesn't allow transactions to already be in the mempool. - CHECK_NONFATAL(tx_result.m_result_type != MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); - if (tx_result.m_result_type == MempoolAcceptResult::ResultType::VALID) { - const CAmount fee = tx_result.m_base_fees.value(); - // Check that fee does not exceed maximum fee - const int64_t virtual_size = tx_result.m_vsize.value(); - const CAmount max_raw_tx_fee = max_raw_tx_fee_rate.GetFee(virtual_size); - if (max_raw_tx_fee && fee > max_raw_tx_fee) { - result_inner.pushKV("allowed", false); - result_inner.pushKV("reject-reason", "max-fee-exceeded"); - exit_early = true; - } else { - // Only return the fee and vsize if the transaction would pass ATMP. - // These can be used to calculate the feerate. - result_inner.pushKV("allowed", true); - result_inner.pushKV("vsize", virtual_size); - UniValue fees(UniValue::VOBJ); - fees.pushKV("base", ValueFromAmount(fee)); - result_inner.pushKV("fees", fees); - } - } else { - result_inner.pushKV("allowed", false); - const TxValidationState state = tx_result.m_state; - if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { - result_inner.pushKV("reject-reason", "missing-inputs"); - } else { - result_inner.pushKV("reject-reason", state.GetRejectReason()); - } - } - rpc_result.push_back(result_inner); - } - return rpc_result; -}, - }; -} - static RPCHelpMan decodepsbt() { return RPCHelpMan{ @@ -1121,6 +733,7 @@ static RPCHelpMan decodepsbt() {RPCResult::Type::OBJ, "scriptPubKey", "", { {RPCResult::Type::STR, "asm", "The asm"}, + {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, {RPCResult::Type::STR_HEX, "hex", "The hex"}, {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, @@ -1255,7 +868,7 @@ static RPCHelpMan decodepsbt() // Add the decoded tx UniValue tx_univ(UniValue::VOBJ); - TxToUniv(CTransaction(*psbtx.tx), uint256(), tx_univ, false); + TxToUniv(CTransaction(*psbtx.tx), /*block_hash=*/uint256(), /*entry=*/tx_univ, /*include_hex=*/false); result.pushKV("tx", tx_univ); // Add the global xpubs @@ -1311,7 +924,7 @@ static RPCHelpMan decodepsbt() txout = input.witness_utxo; UniValue o(UniValue::VOBJ); - ScriptPubKeyToUniv(txout.scriptPubKey, o, /* include_hex */ true); + ScriptToUniv(txout.scriptPubKey, /*out=*/o, /*include_hex=*/true, /*include_address=*/true); UniValue out(UniValue::VOBJ); out.pushKV("amount", ValueFromAmount(txout.nValue)); @@ -1325,7 +938,7 @@ static RPCHelpMan decodepsbt() txout = input.non_witness_utxo->vout[psbtx.tx->vin[i].prevout.n]; UniValue non_wit(UniValue::VOBJ); - TxToUniv(*input.non_witness_utxo, uint256(), non_wit, false); + TxToUniv(*input.non_witness_utxo, /*block_hash=*/uint256(), /*entry=*/non_wit, /*include_hex=*/false); in.pushKV("non_witness_utxo", non_wit); have_a_utxo = true; @@ -1358,12 +971,12 @@ static RPCHelpMan decodepsbt() // Redeem script and witness script if (!input.redeem_script.empty()) { UniValue r(UniValue::VOBJ); - ScriptToUniv(input.redeem_script, r); + ScriptToUniv(input.redeem_script, /*out=*/r); in.pushKV("redeem_script", r); } if (!input.witness_script.empty()) { UniValue r(UniValue::VOBJ); - ScriptToUniv(input.witness_script, r); + ScriptToUniv(input.witness_script, /*out=*/r); in.pushKV("witness_script", r); } @@ -1468,12 +1081,12 @@ static RPCHelpMan decodepsbt() // Redeem script and witness script if (!output.redeem_script.empty()) { UniValue r(UniValue::VOBJ); - ScriptToUniv(output.redeem_script, r); + ScriptToUniv(output.redeem_script, /*out=*/r); out.pushKV("redeem_script", r); } if (!output.witness_script.empty()) { UniValue r(UniValue::VOBJ); - ScriptToUniv(output.witness_script, r); + ScriptToUniv(output.witness_script, /*out=*/r); out.pushKV("witness_script", r); } @@ -2067,33 +1680,24 @@ static RPCHelpMan analyzepsbt() }; } -void RegisterRawTransactionRPCCommands(CRPCTable &t) +void RegisterRawTransactionRPCCommands(CRPCTable& t) { -// clang-format off -static const CRPCCommand commands[] = -{ // category actor (function) - // --------------------- ----------------------- - { "rawtransactions", &getrawtransaction, }, - { "rawtransactions", &createrawtransaction, }, - { "rawtransactions", &decoderawtransaction, }, - { "rawtransactions", &decodescript, }, - { "rawtransactions", &sendrawtransaction, }, - { "rawtransactions", &combinerawtransaction, }, - { "rawtransactions", &signrawtransactionwithkey, }, - { "rawtransactions", &testmempoolaccept, }, - { "rawtransactions", &decodepsbt, }, - { "rawtransactions", &combinepsbt, }, - { "rawtransactions", &finalizepsbt, }, - { "rawtransactions", &createpsbt, }, - { "rawtransactions", &converttopsbt, }, - { "rawtransactions", &utxoupdatepsbt, }, - { "rawtransactions", &joinpsbts, }, - { "rawtransactions", &analyzepsbt, }, - - { "blockchain", &gettxoutproof, }, - { "blockchain", &verifytxoutproof, }, -}; -// clang-format on + static const CRPCCommand commands[]{ + {"rawtransactions", &getrawtransaction}, + {"rawtransactions", &createrawtransaction}, + {"rawtransactions", &decoderawtransaction}, + {"rawtransactions", &decodescript}, + {"rawtransactions", &combinerawtransaction}, + {"rawtransactions", &signrawtransactionwithkey}, + {"rawtransactions", &decodepsbt}, + {"rawtransactions", &combinepsbt}, + {"rawtransactions", &finalizepsbt}, + {"rawtransactions", &createpsbt}, + {"rawtransactions", &converttopsbt}, + {"rawtransactions", &utxoupdatepsbt}, + {"rawtransactions", &joinpsbts}, + {"rawtransactions", &analyzepsbt}, + }; for (const auto& c : commands) { t.appendCommand(c.name, &c); } diff --git a/src/rpc/register.h b/src/rpc/register.h index c5055cc9d7..301f410da3 100644 --- a/src/rpc/register.h +++ b/src/rpc/register.h @@ -9,29 +9,33 @@ * headers for everything under src/rpc/ */ class CRPCTable; -/** Register block chain RPC commands */ void RegisterBlockchainRPCCommands(CRPCTable &tableRPC); -/** Register P2P networking RPC commands */ -void RegisterNetRPCCommands(CRPCTable &tableRPC); -/** Register miscellaneous RPC commands */ -void RegisterMiscRPCCommands(CRPCTable &tableRPC); -/** Register mining RPC commands */ +void RegisterFeeRPCCommands(CRPCTable&); +void RegisterMempoolRPCCommands(CRPCTable&); void RegisterMiningRPCCommands(CRPCTable &tableRPC); -/** Register raw transaction RPC commands */ +void RegisterNodeRPCCommands(CRPCTable&); +void RegisterNetRPCCommands(CRPCTable&); +void RegisterOutputScriptRPCCommands(CRPCTable&); void RegisterRawTransactionRPCCommands(CRPCTable &tableRPC); -/** Register raw transaction RPC commands */ +void RegisterSignMessageRPCCommands(CRPCTable&); void RegisterSignerRPCCommands(CRPCTable &tableRPC); +void RegisterTxoutProofRPCCommands(CRPCTable&); static inline void RegisterAllCoreRPCCommands(CRPCTable &t) { RegisterBlockchainRPCCommands(t); - RegisterNetRPCCommands(t); - RegisterMiscRPCCommands(t); + RegisterFeeRPCCommands(t); + RegisterMempoolRPCCommands(t); RegisterMiningRPCCommands(t); + RegisterNodeRPCCommands(t); + RegisterNetRPCCommands(t); + RegisterOutputScriptRPCCommands(t); RegisterRawTransactionRPCCommands(t); + RegisterSignMessageRPCCommands(t); #ifdef ENABLE_EXTERNAL_SIGNER RegisterSignerRPCCommands(t); #endif // ENABLE_EXTERNAL_SIGNER + RegisterTxoutProofRPCCommands(t); } #endif // BITCOIN_RPC_REGISTER_H diff --git a/src/rpc/request.cpp b/src/rpc/request.cpp index 95a7c25b93..d0e068de19 100644 --- a/src/rpc/request.cpp +++ b/src/rpc/request.cpp @@ -82,7 +82,7 @@ bool GenerateAuthCookie(std::string *cookie_out) { const size_t COOKIE_SIZE = 32; unsigned char rand_pwd[COOKIE_SIZE]; - GetRandBytes(rand_pwd, COOKIE_SIZE); + GetRandBytes(rand_pwd); std::string cookie = COOKIEAUTH_USER + ":" + HexStr(rand_pwd); /** the umask determines what permissions are used to create this file - diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index c70236cc1c..3817a4a45c 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -9,10 +9,9 @@ #include <shutdown.h> #include <sync.h> #include <util/strencodings.h> +#include <util/string.h> #include <util/system.h> -#include <boost/algorithm/string/classification.hpp> -#include <boost/algorithm/string/split.hpp> #include <boost/signals2/signal.hpp> #include <cassert> @@ -167,7 +166,7 @@ static RPCHelpMan stop() // to the client (intended for testing) "\nRequest a graceful shutdown of " PACKAGE_NAME ".", { - {"wait", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "how long to wait in ms", "", {}, /* hidden */ true}, + {"wait", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "how long to wait in ms", "", {}, /*hidden=*/true}, }, RPCResult{RPCResult::Type::STR, "", "A string with the content '" + RESULT + "'"}, RPCExamples{""}, @@ -248,17 +247,13 @@ static RPCHelpMan getrpcinfo() }; } -// clang-format off -static const CRPCCommand vRPCCommands[] = -{ // category actor (function) - // --------------------- ----------------------- +static const CRPCCommand vRPCCommands[]{ /* Overall control/query calls */ - { "control", &getrpcinfo, }, - { "control", &help, }, - { "control", &stop, }, - { "control", &uptime, }, + {"control", &getrpcinfo}, + {"control", &help}, + {"control", &stop}, + {"control", &uptime}, }; -// clang-format on CRPCTable::CRPCTable() { @@ -407,8 +402,7 @@ static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, c // Process expected parameters. int hole = 0; for (const std::string &argNamePattern: argNames) { - std::vector<std::string> vargNames; - boost::algorithm::split(vargNames, argNamePattern, boost::algorithm::is_any_of("|")); + std::vector<std::string> vargNames = SplitString(argNamePattern, '|'); auto fr = argsIn.end(); for (const std::string & argName : vargNames) { fr = argsIn.find(argName); diff --git a/src/rpc/signmessage.cpp b/src/rpc/signmessage.cpp new file mode 100644 index 0000000000..8c752ba1fd --- /dev/null +++ b/src/rpc/signmessage.cpp @@ -0,0 +1,113 @@ +// Copyright (c) 2010 Satoshi Nakamoto +// Copyright (c) 2009-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <key.h> +#include <key_io.h> +#include <rpc/protocol.h> +#include <rpc/request.h> +#include <rpc/server.h> +#include <rpc/util.h> +#include <univalue.h> +#include <util/message.h> + +#include <string> + +static RPCHelpMan verifymessage() +{ + return RPCHelpMan{"verifymessage", + "Verify a signed message.", + { + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address to use for the signature."}, + {"signature", RPCArg::Type::STR, RPCArg::Optional::NO, "The signature provided by the signer in base 64 encoding (see signmessage)."}, + {"message", RPCArg::Type::STR, RPCArg::Optional::NO, "The message that was signed."}, + }, + RPCResult{ + RPCResult::Type::BOOL, "", "If the signature is verified or not." + }, + RPCExamples{ + "\nUnlock the wallet for 30 seconds\n" + + HelpExampleCli("walletpassphrase", "\"mypassphrase\" 30") + + "\nCreate the signature\n" + + HelpExampleCli("signmessage", "\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\" \"my message\"") + + "\nVerify the signature\n" + + HelpExampleCli("verifymessage", "\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\" \"signature\" \"my message\"") + + "\nAs a JSON-RPC call\n" + + HelpExampleRpc("verifymessage", "\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\", \"signature\", \"my message\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + std::string strAddress = request.params[0].get_str(); + std::string strSign = request.params[1].get_str(); + std::string strMessage = request.params[2].get_str(); + + switch (MessageVerify(strAddress, strSign, strMessage)) { + case MessageVerificationResult::ERR_INVALID_ADDRESS: + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + case MessageVerificationResult::ERR_ADDRESS_NO_KEY: + throw JSONRPCError(RPC_TYPE_ERROR, "Address does not refer to key"); + case MessageVerificationResult::ERR_MALFORMED_SIGNATURE: + throw JSONRPCError(RPC_TYPE_ERROR, "Malformed base64 encoding"); + case MessageVerificationResult::ERR_PUBKEY_NOT_RECOVERED: + case MessageVerificationResult::ERR_NOT_SIGNED: + return false; + case MessageVerificationResult::OK: + return true; + } + + return false; + }, + }; +} + +static RPCHelpMan signmessagewithprivkey() +{ + return RPCHelpMan{"signmessagewithprivkey", + "\nSign a message with the private key of an address\n", + { + {"privkey", RPCArg::Type::STR, RPCArg::Optional::NO, "The private key to sign the message with."}, + {"message", RPCArg::Type::STR, RPCArg::Optional::NO, "The message to create a signature of."}, + }, + RPCResult{ + RPCResult::Type::STR, "signature", "The signature of the message encoded in base 64" + }, + RPCExamples{ + "\nCreate the signature\n" + + HelpExampleCli("signmessagewithprivkey", "\"privkey\" \"my message\"") + + "\nVerify the signature\n" + + HelpExampleCli("verifymessage", "\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\" \"signature\" \"my message\"") + + "\nAs a JSON-RPC call\n" + + HelpExampleRpc("signmessagewithprivkey", "\"privkey\", \"my message\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + std::string strPrivkey = request.params[0].get_str(); + std::string strMessage = request.params[1].get_str(); + + CKey key = DecodeSecret(strPrivkey); + if (!key.IsValid()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key"); + } + + std::string signature; + + if (!MessageSign(key, strMessage, signature)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Sign failed"); + } + + return signature; + }, + }; +} + +void RegisterSignMessageRPCCommands(CRPCTable& t) +{ + static const CRPCCommand commands[]{ + {"util", &verifymessage}, + {"util", &signmessagewithprivkey}, + }; + for (const auto& c : commands) { + t.appendCommand(c.name, &c); + } +} diff --git a/src/rpc/txoutproof.cpp b/src/rpc/txoutproof.cpp new file mode 100644 index 0000000000..d16820baeb --- /dev/null +++ b/src/rpc/txoutproof.cpp @@ -0,0 +1,181 @@ +// Copyright (c) 2010 Satoshi Nakamoto +// Copyright (c) 2009-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chain.h> +#include <chainparams.h> +#include <coins.h> +#include <index/txindex.h> +#include <merkleblock.h> +#include <node/blockstorage.h> +#include <primitives/transaction.h> +#include <rpc/server.h> +#include <rpc/server_util.h> +#include <rpc/util.h> +#include <univalue.h> +#include <util/strencodings.h> +#include <validation.h> + +using node::GetTransaction; +using node::ReadBlockFromDisk; + +static RPCHelpMan gettxoutproof() +{ + return RPCHelpMan{"gettxoutproof", + "\nReturns a hex-encoded proof that \"txid\" was included in a block.\n" + "\nNOTE: By default this function only works sometimes. This is when there is an\n" + "unspent output in the utxo for this transaction. To make it always work,\n" + "you need to maintain a transaction index, using the -txindex command line option or\n" + "specify the block in which the transaction is included manually (by blockhash).\n", + { + {"txids", RPCArg::Type::ARR, RPCArg::Optional::NO, "The txids to filter", + { + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A transaction hash"}, + }, + }, + {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED_NAMED_ARG, "If specified, looks for txid in the block with this hash"}, + }, + RPCResult{ + RPCResult::Type::STR, "data", "A string that is a serialized, hex-encoded data for the proof." + }, + RPCExamples{""}, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + std::set<uint256> setTxids; + UniValue txids = request.params[0].get_array(); + if (txids.empty()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter 'txids' cannot be empty"); + } + for (unsigned int idx = 0; idx < txids.size(); idx++) { + auto ret = setTxids.insert(ParseHashV(txids[idx], "txid")); + if (!ret.second) { + throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid parameter, duplicated txid: ") + txids[idx].get_str()); + } + } + + const CBlockIndex* pblockindex = nullptr; + uint256 hashBlock; + ChainstateManager& chainman = EnsureAnyChainman(request.context); + if (!request.params[1].isNull()) { + LOCK(cs_main); + hashBlock = ParseHashV(request.params[1], "blockhash"); + pblockindex = chainman.m_blockman.LookupBlockIndex(hashBlock); + if (!pblockindex) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + } + } else { + LOCK(cs_main); + CChainState& active_chainstate = chainman.ActiveChainstate(); + + // Loop through txids and try to find which block they're in. Exit loop once a block is found. + for (const auto& tx : setTxids) { + const Coin& coin = AccessByTxid(active_chainstate.CoinsTip(), tx); + if (!coin.IsSpent()) { + pblockindex = active_chainstate.m_chain[coin.nHeight]; + break; + } + } + } + + + // Allow txindex to catch up if we need to query it and before we acquire cs_main. + if (g_txindex && !pblockindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + + LOCK(cs_main); + + if (pblockindex == nullptr) { + const CTransactionRef tx = GetTransaction(/*block_index=*/nullptr, /*mempool=*/nullptr, *setTxids.begin(), Params().GetConsensus(), hashBlock); + if (!tx || hashBlock.IsNull()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not yet in block"); + } + pblockindex = chainman.m_blockman.LookupBlockIndex(hashBlock); + if (!pblockindex) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Transaction index corrupt"); + } + } + + CBlock block; + if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Can't read block from disk"); + } + + unsigned int ntxFound = 0; + for (const auto& tx : block.vtx) { + if (setTxids.count(tx->GetHash())) { + ntxFound++; + } + } + if (ntxFound != setTxids.size()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Not all transactions found in specified or retrieved block"); + } + + CDataStream ssMB(SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS); + CMerkleBlock mb(block, setTxids); + ssMB << mb; + std::string strHex = HexStr(ssMB); + return strHex; + }, + }; +} + +static RPCHelpMan verifytxoutproof() +{ + return RPCHelpMan{"verifytxoutproof", + "\nVerifies that a proof points to a transaction in a block, returning the transaction it commits to\n" + "and throwing an RPC error if the block is not in our best chain\n", + { + {"proof", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex-encoded proof generated by gettxoutproof"}, + }, + RPCResult{ + RPCResult::Type::ARR, "", "", + { + {RPCResult::Type::STR_HEX, "txid", "The txid(s) which the proof commits to, or empty array if the proof cannot be validated."}, + } + }, + RPCExamples{""}, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + CDataStream ssMB(ParseHexV(request.params[0], "proof"), SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS); + CMerkleBlock merkleBlock; + ssMB >> merkleBlock; + + UniValue res(UniValue::VARR); + + std::vector<uint256> vMatch; + std::vector<unsigned int> vIndex; + if (merkleBlock.txn.ExtractMatches(vMatch, vIndex) != merkleBlock.header.hashMerkleRoot) + return res; + + ChainstateManager& chainman = EnsureAnyChainman(request.context); + LOCK(cs_main); + + const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(merkleBlock.header.GetHash()); + if (!pindex || !chainman.ActiveChain().Contains(pindex) || pindex->nTx == 0) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found in chain"); + } + + // Check if proof is valid, only add results if so + if (pindex->nTx == merkleBlock.txn.GetNumTransactions()) { + for (const uint256& hash : vMatch) { + res.push_back(hash.GetHex()); + } + } + + return res; + }, + }; +} + +void RegisterTxoutProofRPCCommands(CRPCTable& t) +{ + static const CRPCCommand commands[]{ + {"blockchain", &gettxoutproof}, + {"blockchain", &verifytxoutproof}, + }; + for (const auto& c : commands) { + t.appendCommand(c.name, &c); + } +} diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 7c859268be..ef3e58494e 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -9,15 +9,13 @@ #include <script/descriptor.h> #include <script/signingprovider.h> #include <tinyformat.h> +#include <util/check.h> #include <util/strencodings.h> #include <util/string.h> #include <util/translation.h> #include <tuple> -#include <boost/algorithm/string/classification.hpp> -#include <boost/algorithm/string/split.hpp> - const std::string UNIX_EPOCH_TIME = "UNIX epoch time"; const std::string EXAMPLE_ADDRESS[2] = {"bc1q09vm5lfy0j5reeulh4x5752q25uqqvz34hufdl", "bc1q02ad21edsxd23d32dfgqqsz4vv4nmtfzuklhy3"}; @@ -421,7 +419,7 @@ struct Sections { if (arg.m_type_str.size() != 0 && push_name) { left += "\"" + arg.GetName() + "\": " + arg.m_type_str.at(0); } else { - left += push_name ? arg.ToStringObj(/* oneline */ false) : arg.ToString(/* oneline */ false); + left += push_name ? arg.ToStringObj(/*oneline=*/false) : arg.ToString(/*oneline=*/false); } left += ","; PushSection({left, arg.ToDescriptionString()}); @@ -513,8 +511,7 @@ RPCHelpMan::RPCHelpMan(std::string name, std::string description, std::vector<RP { std::set<std::string> named_args; for (const auto& arg : m_args) { - std::vector<std::string> names; - boost::split(names, arg.m_names, boost::is_any_of("|")); + std::vector<std::string> names = SplitString(arg.m_names, '|'); // Should have unique named arguments for (const std::string& name : names) { CHECK_NONFATAL(named_args.insert(name).second); @@ -542,7 +539,7 @@ RPCHelpMan::RPCHelpMan(std::string name, std::string description, std::vector<RP // Null values are accepted in all arguments break; default: - CHECK_NONFATAL(false); + NONFATAL_UNREACHABLE(); break; } } @@ -627,7 +624,7 @@ std::string RPCHelpMan::ToString() const if (was_optional) ret += ") "; was_optional = false; } - ret += arg.ToString(/* oneline */ true); + ret += arg.ToString(/*oneline=*/true); } if (was_optional) ret += " )"; @@ -665,8 +662,7 @@ UniValue RPCHelpMan::GetArgMap() const UniValue arr{UniValue::VARR}; for (int i{0}; i < int(m_args.size()); ++i) { const auto& arg = m_args.at(i); - std::vector<std::string> arg_names; - boost::split(arg_names, arg.m_names, boost::is_any_of("|")); + std::vector<std::string> arg_names = SplitString(arg.m_names, '|'); for (const auto& arg_name : arg_names) { UniValue map{UniValue::VARR}; map.push_back(m_name); @@ -774,7 +770,7 @@ void RPCResult::ToSections(Sections& sections, const OuterType outer_type, const // Elements in a JSON structure (dictionary or array) are separated by a comma const std::string maybe_separator{outer_type != OuterType::NONE ? "," : ""}; - // The key name if recursed into an dictionary + // The key name if recursed into a dictionary const std::string maybe_key{ outer_type == OuterType::OBJ ? "\"" + this->m_key_name + "\" : " : @@ -793,7 +789,7 @@ void RPCResult::ToSections(Sections& sections, const OuterType outer_type, const return; } case Type::ANY: { - CHECK_NONFATAL(false); // Only for testing + NONFATAL_UNREACHABLE(); // Only for testing } case Type::NONE: { sections.PushSection({indent + "null" + maybe_separator, Description("json null")}); @@ -860,15 +856,16 @@ void RPCResult::ToSections(Sections& sections, const OuterType outer_type, const return; } } // no default case, so the compiler can warn about missing cases - CHECK_NONFATAL(false); + NONFATAL_UNREACHABLE(); } bool RPCResult::MatchesType(const UniValue& result) const { - switch (m_type) { - case Type::ELISION: { - return false; + if (m_skip_type_check) { + return true; } + switch (m_type) { + case Type::ELISION: case Type::ANY: { return true; } @@ -889,14 +886,55 @@ bool RPCResult::MatchesType(const UniValue& result) const } case Type::ARR_FIXED: case Type::ARR: { - return UniValue::VARR == result.getType(); + if (UniValue::VARR != result.getType()) return false; + for (size_t i{0}; i < result.get_array().size(); ++i) { + // If there are more results than documented, re-use the last doc_inner. + const RPCResult& doc_inner{m_inner.at(std::min(m_inner.size() - 1, i))}; + if (!doc_inner.MatchesType(result.get_array()[i])) return false; + } + return true; // empty result array is valid } case Type::OBJ_DYN: case Type::OBJ: { - return UniValue::VOBJ == result.getType(); + if (UniValue::VOBJ != result.getType()) return false; + if (!m_inner.empty() && m_inner.at(0).m_type == Type::ELISION) return true; + if (m_type == Type::OBJ_DYN) { + const RPCResult& doc_inner{m_inner.at(0)}; // Assume all types are the same, randomly pick the first + for (size_t i{0}; i < result.get_obj().size(); ++i) { + if (!doc_inner.MatchesType(result.get_obj()[i])) { + return false; + } + } + return true; // empty result obj is valid + } + std::set<std::string> doc_keys; + for (const auto& doc_entry : m_inner) { + doc_keys.insert(doc_entry.m_key_name); + } + std::map<std::string, UniValue> result_obj; + result.getObjMap(result_obj); + for (const auto& result_entry : result_obj) { + if (doc_keys.find(result_entry.first) == doc_keys.end()) { + return false; // missing documentation + } + } + + for (const auto& doc_entry : m_inner) { + const auto result_it{result_obj.find(doc_entry.m_key_name)}; + if (result_it == result_obj.end()) { + if (!doc_entry.m_optional) { + return false; // result is missing a required key + } + continue; + } + if (!doc_entry.MatchesType(result_it->second)) { + return false; // wrong type + } + } + return true; } } // no default case, so the compiler can warn about missing cases - CHECK_NONFATAL(false); + NONFATAL_UNREACHABLE(); } void RPCResult::CheckInnerDoc() const @@ -942,9 +980,9 @@ std::string RPCArg::ToStringObj(const bool oneline) const case Type::OBJ: case Type::OBJ_USER_KEYS: // Currently unused, so avoid writing dead code - CHECK_NONFATAL(false); + NONFATAL_UNREACHABLE(); } // no default case, so the compiler can warn about missing cases - CHECK_NONFATAL(false); + NONFATAL_UNREACHABLE(); } std::string RPCArg::ToString(const bool oneline) const @@ -979,7 +1017,7 @@ std::string RPCArg::ToString(const bool oneline) const return "[" + res + "...]"; } } // no default case, so the compiler can warn about missing cases - CHECK_NONFATAL(false); + NONFATAL_UNREACHABLE(); } static std::pair<int64_t, int64_t> ParseRange(const UniValue& value) diff --git a/src/rpc/util.h b/src/rpc/util.h index 89d32d4193..e16fed75bc 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -256,6 +256,7 @@ struct RPCResult { const std::string m_key_name; //!< Only used for dicts const std::vector<RPCResult> m_inner; //!< Only used for arrays or dicts const bool m_optional; + const bool m_skip_type_check; const std::string m_description; const std::string m_cond; @@ -270,6 +271,7 @@ struct RPCResult { m_key_name{std::move(m_key_name)}, m_inner{std::move(inner)}, m_optional{optional}, + m_skip_type_check{false}, m_description{std::move(description)}, m_cond{std::move(cond)} { @@ -290,11 +292,13 @@ struct RPCResult { const std::string m_key_name, const bool optional, const std::string description, - const std::vector<RPCResult> inner = {}) + const std::vector<RPCResult> inner = {}, + bool skip_type_check = false) : m_type{std::move(type)}, m_key_name{std::move(m_key_name)}, m_inner{std::move(inner)}, m_optional{optional}, + m_skip_type_check{skip_type_check}, m_description{std::move(description)}, m_cond{} { @@ -305,8 +309,9 @@ struct RPCResult { const Type type, const std::string m_key_name, const std::string description, - const std::vector<RPCResult> inner = {}) - : RPCResult{type, m_key_name, false, description, inner} {} + const std::vector<RPCResult> inner = {}, + bool skip_type_check = false) + : RPCResult{type, m_key_name, false, description, inner, skip_type_check} {} /** Append the sections of the result. */ void ToSections(Sections& sections, OuterType outer_type = OuterType::NONE, const int current_indent = 0) const; diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 0b2ad3c553..570b417ff5 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -4,11 +4,11 @@ #include <scheduler.h> -#include <random.h> +#include <sync.h> #include <util/syscall_sandbox.h> #include <util/time.h> -#include <assert.h> +#include <cassert> #include <functional> #include <utility> @@ -43,7 +43,7 @@ void CScheduler::serviceQueue() // the time of the first item on the queue: while (!shouldStop() && !taskQueue.empty()) { - std::chrono::system_clock::time_point timeToWaitFor = taskQueue.begin()->first; + std::chrono::steady_clock::time_point timeToWaitFor = taskQueue.begin()->first; if (newTaskScheduled.wait_until(lock, timeToWaitFor) == std::cv_status::timeout) { break; // Exit loop after timeout, it means we reached the time of the event } @@ -72,7 +72,7 @@ void CScheduler::serviceQueue() newTaskScheduled.notify_one(); } -void CScheduler::schedule(CScheduler::Function f, std::chrono::system_clock::time_point t) +void CScheduler::schedule(CScheduler::Function f, std::chrono::steady_clock::time_point t) { { LOCK(newTaskMutex); @@ -89,7 +89,7 @@ void CScheduler::MockForward(std::chrono::seconds delta_seconds) LOCK(newTaskMutex); // use temp_queue to maintain updated schedule - std::multimap<std::chrono::system_clock::time_point, Function> temp_queue; + std::multimap<std::chrono::steady_clock::time_point, Function> temp_queue; for (const auto& element : taskQueue) { temp_queue.emplace_hint(temp_queue.cend(), element.first - delta_seconds, element.second); @@ -111,11 +111,11 @@ static void Repeat(CScheduler& s, CScheduler::Function f, std::chrono::milliseco void CScheduler::scheduleEvery(CScheduler::Function f, std::chrono::milliseconds delta) { - scheduleFromNow([=] { Repeat(*this, f, delta); }, delta); + scheduleFromNow([this, f, delta] { Repeat(*this, f, delta); }, delta); } -size_t CScheduler::getQueueInfo(std::chrono::system_clock::time_point& first, - std::chrono::system_clock::time_point& last) const +size_t CScheduler::getQueueInfo(std::chrono::steady_clock::time_point& first, + std::chrono::steady_clock::time_point& last) const { LOCK(newTaskMutex); size_t result = taskQueue.size(); @@ -143,7 +143,7 @@ void SingleThreadedSchedulerClient::MaybeScheduleProcessQueue() if (m_are_callbacks_running) return; if (m_callbacks_pending.empty()) return; } - m_pscheduler->schedule(std::bind(&SingleThreadedSchedulerClient::ProcessQueue, this), std::chrono::system_clock::now()); + m_scheduler.schedule([this] { this->ProcessQueue(); }, std::chrono::steady_clock::now()); } void SingleThreadedSchedulerClient::ProcessQueue() @@ -179,8 +179,6 @@ void SingleThreadedSchedulerClient::ProcessQueue() void SingleThreadedSchedulerClient::AddToProcessQueue(std::function<void()> func) { - assert(m_pscheduler); - { LOCK(m_callbacks_mutex); m_callbacks_pending.emplace_back(std::move(func)); @@ -190,7 +188,7 @@ void SingleThreadedSchedulerClient::AddToProcessQueue(std::function<void()> func void SingleThreadedSchedulerClient::EmptyQueue() { - assert(!m_pscheduler->AreThreadsServicingQueue()); + assert(!m_scheduler.AreThreadsServicingQueue()); bool should_continue = true; while (should_continue) { ProcessQueue(); diff --git a/src/scheduler.h b/src/scheduler.h index bb0abfbf7a..b8245f97ed 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -5,13 +5,18 @@ #ifndef BITCOIN_SCHEDULER_H #define BITCOIN_SCHEDULER_H +#include <attributes.h> +#include <sync.h> +#include <threadsafety.h> + +#include <chrono> #include <condition_variable> +#include <cstddef> #include <functional> #include <list> #include <map> #include <thread> - -#include <sync.h> +#include <utility> /** * Simple class for background tasks that should be run @@ -41,12 +46,12 @@ public: typedef std::function<void()> Function; /** Call func at/after time t */ - void schedule(Function f, std::chrono::system_clock::time_point t); + void schedule(Function f, std::chrono::steady_clock::time_point t); /** Call f once after the delta has passed */ void scheduleFromNow(Function f, std::chrono::milliseconds delta) { - schedule(std::move(f), std::chrono::system_clock::now() + delta); + schedule(std::move(f), std::chrono::steady_clock::now() + delta); } /** @@ -88,8 +93,8 @@ public: * Returns number of tasks waiting to be serviced, * and first and last task times */ - size_t getQueueInfo(std::chrono::system_clock::time_point& first, - std::chrono::system_clock::time_point& last) const; + size_t getQueueInfo(std::chrono::steady_clock::time_point& first, + std::chrono::steady_clock::time_point& last) const; /** Returns true if there are threads actively running in serviceQueue() */ bool AreThreadsServicingQueue() const; @@ -97,7 +102,7 @@ public: private: mutable Mutex newTaskMutex; std::condition_variable newTaskScheduled; - std::multimap<std::chrono::system_clock::time_point, Function> taskQueue GUARDED_BY(newTaskMutex); + std::multimap<std::chrono::steady_clock::time_point, Function> taskQueue GUARDED_BY(newTaskMutex); int nThreadsServicingQueue GUARDED_BY(newTaskMutex){0}; bool stopRequested GUARDED_BY(newTaskMutex){false}; bool stopWhenEmpty GUARDED_BY(newTaskMutex){false}; @@ -117,7 +122,7 @@ private: class SingleThreadedSchedulerClient { private: - CScheduler* m_pscheduler; + CScheduler& m_scheduler; Mutex m_callbacks_mutex; std::list<std::function<void()>> m_callbacks_pending GUARDED_BY(m_callbacks_mutex); @@ -127,7 +132,7 @@ private: void ProcessQueue(); public: - explicit SingleThreadedSchedulerClient(CScheduler* pschedulerIn) : m_pscheduler(pschedulerIn) {} + explicit SingleThreadedSchedulerClient(CScheduler& scheduler LIFETIMEBOUND) : m_scheduler{scheduler} {} /** * Add a callback to be executed. Callbacks are executed serially diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 798c4b3ea0..cece0b60ce 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -802,6 +802,30 @@ public: bool IsSingleType() const final { return true; } }; +/** A parsed (sorted)multi_a(...) descriptor. Always uses x-only pubkeys. */ +class MultiADescriptor final : public DescriptorImpl +{ + const int m_threshold; + const bool m_sorted; +protected: + std::string ToStringExtra() const override { return strprintf("%i", m_threshold); } + std::vector<CScript> MakeScripts(const std::vector<CPubKey>& keys, Span<const CScript>, FlatSigningProvider&) const override { + CScript ret; + std::vector<XOnlyPubKey> xkeys; + for (const auto& key : keys) xkeys.emplace_back(key); + if (m_sorted) std::sort(xkeys.begin(), xkeys.end()); + ret << ToByteVector(xkeys[0]) << OP_CHECKSIG; + for (size_t i = 1; i < keys.size(); ++i) { + ret << ToByteVector(xkeys[i]) << OP_CHECKSIGADD; + } + ret << m_threshold << OP_NUMEQUAL; + return Vector(std::move(ret)); + } +public: + MultiADescriptor(int threshold, std::vector<std::unique_ptr<PubkeyProvider>> providers, bool sorted = false) : DescriptorImpl(std::move(providers), sorted ? "sortedmulti_a" : "multi_a"), m_threshold(threshold), m_sorted(sorted) {} + bool IsSingleType() const final { return true; } +}; + /** A parsed sh(...) descriptor. */ class SHDescriptor final : public DescriptorImpl { @@ -1040,16 +1064,21 @@ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const using namespace spanparsing; auto expr = Expr(sp); - bool sorted_multi = false; if (Func("pk", expr)) { auto pubkey = ParsePubkey(key_exp_index, expr, ctx, out, error); - if (!pubkey) return nullptr; + if (!pubkey) { + error = strprintf("pk(): %s", error); + return nullptr; + } ++key_exp_index; return std::make_unique<PKDescriptor>(std::move(pubkey), ctx == ParseScriptContext::P2TR); } if ((ctx == ParseScriptContext::TOP || ctx == ParseScriptContext::P2SH || ctx == ParseScriptContext::P2WSH) && Func("pkh", expr)) { auto pubkey = ParsePubkey(key_exp_index, expr, ctx, out, error); - if (!pubkey) return nullptr; + if (!pubkey) { + error = strprintf("pkh(): %s", error); + return nullptr; + } ++key_exp_index; return std::make_unique<PKHDescriptor>(std::move(pubkey)); } else if (Func("pkh", expr)) { @@ -1058,14 +1087,22 @@ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const } if (ctx == ParseScriptContext::TOP && Func("combo", expr)) { auto pubkey = ParsePubkey(key_exp_index, expr, ctx, out, error); - if (!pubkey) return nullptr; + if (!pubkey) { + error = strprintf("combo(): %s", error); + return nullptr; + } ++key_exp_index; return std::make_unique<ComboDescriptor>(std::move(pubkey)); } else if (Func("combo", expr)) { error = "Can only have combo() at top level"; return nullptr; } - if ((ctx == ParseScriptContext::TOP || ctx == ParseScriptContext::P2SH || ctx == ParseScriptContext::P2WSH) && ((sorted_multi = Func("sortedmulti", expr)) || Func("multi", expr))) { + const bool multi = Func("multi", expr); + const bool sortedmulti = !multi && Func("sortedmulti", expr); + const bool multi_a = !(multi || sortedmulti) && Func("multi_a", expr); + const bool sortedmulti_a = !(multi || sortedmulti || multi_a) && Func("sortedmulti_a", expr); + if (((ctx == ParseScriptContext::TOP || ctx == ParseScriptContext::P2SH || ctx == ParseScriptContext::P2WSH) && (multi || sortedmulti)) || + (ctx == ParseScriptContext::P2TR && (multi_a || sortedmulti_a))) { auto threshold = Expr(expr); uint32_t thres; std::vector<std::unique_ptr<PubkeyProvider>> providers; @@ -1081,14 +1118,20 @@ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const } auto arg = Expr(expr); auto pk = ParsePubkey(key_exp_index, arg, ctx, out, error); - if (!pk) return nullptr; + if (!pk) { + error = strprintf("Multi: %s", error); + return nullptr; + } script_size += pk->GetSize() + 1; providers.emplace_back(std::move(pk)); key_exp_index++; } - if (providers.empty() || providers.size() > MAX_PUBKEYS_PER_MULTISIG) { + if ((multi || sortedmulti) && (providers.empty() || providers.size() > MAX_PUBKEYS_PER_MULTISIG)) { error = strprintf("Cannot have %u keys in multisig; must have between 1 and %d keys, inclusive", providers.size(), MAX_PUBKEYS_PER_MULTISIG); return nullptr; + } else if ((multi_a || sortedmulti_a) && (providers.empty() || providers.size() > MAX_PUBKEYS_PER_MULTI_A)) { + error = strprintf("Cannot have %u keys in multi_a; must have between 1 and %d keys, inclusive", providers.size(), MAX_PUBKEYS_PER_MULTI_A); + return nullptr; } else if (thres < 1) { error = strprintf("Multisig threshold cannot be %d, must be at least 1", thres); return nullptr; @@ -1109,14 +1152,24 @@ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const return nullptr; } } - return std::make_unique<MultisigDescriptor>(thres, std::move(providers), sorted_multi); - } else if (Func("sortedmulti", expr) || Func("multi", expr)) { + if (multi || sortedmulti) { + return std::make_unique<MultisigDescriptor>(thres, std::move(providers), sortedmulti); + } else { + return std::make_unique<MultiADescriptor>(thres, std::move(providers), sortedmulti_a); + } + } else if (multi || sortedmulti) { error = "Can only have multi/sortedmulti at top level, in sh(), or in wsh()"; return nullptr; + } else if (multi_a || sortedmulti_a) { + error = "Can only have multi_a/sortedmulti_a inside tr()"; + return nullptr; } if ((ctx == ParseScriptContext::TOP || ctx == ParseScriptContext::P2SH) && Func("wpkh", expr)) { auto pubkey = ParsePubkey(key_exp_index, expr, ParseScriptContext::P2WPKH, out, error); - if (!pubkey) return nullptr; + if (!pubkey) { + error = strprintf("wpkh(): %s", error); + return nullptr; + } key_exp_index++; return std::make_unique<WPKHDescriptor>(std::move(pubkey)); } else if (Func("wpkh", expr)) { @@ -1153,7 +1206,10 @@ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const if (ctx == ParseScriptContext::TOP && Func("tr", expr)) { auto arg = Expr(expr); auto internal_key = ParsePubkey(key_exp_index, arg, ParseScriptContext::P2TR, out, error); - if (!internal_key) return nullptr; + if (!internal_key) { + error = strprintf("tr(): %s", error); + return nullptr; + } ++key_exp_index; std::vector<std::unique_ptr<DescriptorImpl>> subscripts; //!< list of script subexpressions std::vector<int> depths; //!< depth in the tree of each subexpression (same length subscripts) @@ -1257,6 +1313,21 @@ std::unique_ptr<PubkeyProvider> InferXOnlyPubkey(const XOnlyPubKey& xkey, ParseS return key_provider; } +std::unique_ptr<DescriptorImpl> InferMultiA(const CScript& script, ParseScriptContext ctx, const SigningProvider& provider) +{ + auto match = MatchMultiA(script); + if (!match) return {}; + std::vector<std::unique_ptr<PubkeyProvider>> keys; + keys.reserve(match->second.size()); + for (const auto keyspan : match->second) { + if (keyspan.size() != 32) return {}; + auto key = InferXOnlyPubkey(XOnlyPubKey{keyspan}, ctx, provider); + if (!key) return {}; + keys.push_back(std::move(key)); + } + return std::make_unique<MultiADescriptor>(match->first, std::move(keys)); +} + std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptContext ctx, const SigningProvider& provider) { if (ctx == ParseScriptContext::P2TR && script.size() == 34 && script[0] == 32 && script[33] == OP_CHECKSIG) { @@ -1264,6 +1335,11 @@ std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptCo return std::make_unique<PKDescriptor>(InferXOnlyPubkey(key, ctx, provider), true); } + if (ctx == ParseScriptContext::P2TR) { + auto ret = InferMultiA(script, ctx, provider); + if (ret) return ret; + } + std::vector<std::vector<unsigned char>> data; TxoutType txntype = Solver(script, data); diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 11b1a1c887..c4d13d7283 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -225,31 +225,6 @@ bool static CheckPubKeyEncoding(const valtype &vchPubKey, unsigned int flags, co return true; } -bool CheckMinimalPush(const valtype& data, opcodetype opcode) { - // Excludes OP_1NEGATE, OP_1-16 since they are by definition minimal - assert(0 <= opcode && opcode <= OP_PUSHDATA4); - if (data.size() == 0) { - // Should have used OP_0. - return opcode == OP_0; - } else if (data.size() == 1 && data[0] >= 1 && data[0] <= 16) { - // Should have used OP_1 .. OP_16. - return false; - } else if (data.size() == 1 && data[0] == 0x81) { - // Should have used OP_1NEGATE. - return false; - } else if (data.size() <= 75) { - // Must have used a direct push (opcode indicating number of bytes pushed + those bytes). - return opcode == data.size(); - } else if (data.size() <= 255) { - // Must have used OP_PUSHDATA. - return opcode == OP_PUSHDATA1; - } else if (data.size() <= 65535) { - // Must have used OP_PUSHDATA2. - return opcode == OP_PUSHDATA2; - } - return true; -} - int FindAndDelete(CScript& script, const CScript& b) { int nFound = 0; @@ -2009,7 +1984,7 @@ bool VerifyScript(const CScript& scriptSig, const CScript& scriptPubKey, const C // The scriptSig must be _exactly_ CScript(), otherwise we reintroduce malleability. return set_error(serror, SCRIPT_ERR_WITNESS_MALLEATED); } - if (!VerifyWitnessProgram(*witness, witnessversion, witnessprogram, flags, checker, serror, /* is_p2sh */ false)) { + if (!VerifyWitnessProgram(*witness, witnessversion, witnessprogram, flags, checker, serror, /*is_p2sh=*/false)) { return false; } // Bypass the cleanstack check at the end. The actual stack is obviously not clean @@ -2054,7 +2029,7 @@ bool VerifyScript(const CScript& scriptSig, const CScript& scriptPubKey, const C // reintroduce malleability. return set_error(serror, SCRIPT_ERR_WITNESS_MALLEATED_P2SH); } - if (!VerifyWitnessProgram(*witness, witnessversion, witnessprogram, flags, checker, serror, /* is_p2sh */ true)) { + if (!VerifyWitnessProgram(*witness, witnessversion, witnessprogram, flags, checker, serror, /*is_p2sh=*/true)) { return false; } // Bypass the cleanstack check at the end. The actual stack is obviously not clean diff --git a/src/script/interpreter.h b/src/script/interpreter.h index cf1953ad22..adf454aa15 100644 --- a/src/script/interpreter.h +++ b/src/script/interpreter.h @@ -151,7 +151,7 @@ bool CheckSignatureEncoding(const std::vector<unsigned char> &vchSig, unsigned i struct PrecomputedTransactionData { // BIP341 precomputed data. - // These are single-SHA256, see https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki#cite_note-15. + // These are single-SHA256, see https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki#cite_note-16. uint256 m_prevouts_single_hash; uint256 m_sequences_single_hash; uint256 m_outputs_single_hash; @@ -344,8 +344,6 @@ bool VerifyScript(const CScript& scriptSig, const CScript& scriptPubKey, const C size_t CountWitnessSigOps(const CScript& scriptSig, const CScript& scriptPubKey, const CScriptWitness* witness, unsigned int flags); -bool CheckMinimalPush(const std::vector<unsigned char>& data, opcodetype opcode); - int FindAndDelete(CScript& script, const CScript& b); #endif // BITCOIN_SCRIPT_INTERPRETER_H diff --git a/src/script/miniscript.cpp b/src/script/miniscript.cpp new file mode 100644 index 0000000000..019f02f159 --- /dev/null +++ b/src/script/miniscript.cpp @@ -0,0 +1,349 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <string> +#include <vector> +#include <script/script.h> +#include <script/standard.h> +#include <script/miniscript.h> + +#include <assert.h> + +namespace miniscript { +namespace internal { + +Type SanitizeType(Type e) { + int num_types = (e << "K"_mst) + (e << "V"_mst) + (e << "B"_mst) + (e << "W"_mst); + if (num_types == 0) return ""_mst; // No valid type, don't care about the rest + assert(num_types == 1); // K, V, B, W all conflict with each other + bool ok = // Work around a GCC 4.8 bug that breaks user-defined literals in macro calls. + (!(e << "z"_mst) || !(e << "o"_mst)) && // z conflicts with o + (!(e << "n"_mst) || !(e << "z"_mst)) && // n conflicts with z + (!(e << "n"_mst) || !(e << "W"_mst)) && // n conflicts with W + (!(e << "V"_mst) || !(e << "d"_mst)) && // V conflicts with d + (!(e << "K"_mst) || (e << "u"_mst)) && // K implies u + (!(e << "V"_mst) || !(e << "u"_mst)) && // V conflicts with u + (!(e << "e"_mst) || !(e << "f"_mst)) && // e conflicts with f + (!(e << "e"_mst) || (e << "d"_mst)) && // e implies d + (!(e << "V"_mst) || !(e << "e"_mst)) && // V conflicts with e + (!(e << "d"_mst) || !(e << "f"_mst)) && // d conflicts with f + (!(e << "V"_mst) || (e << "f"_mst)) && // V implies f + (!(e << "K"_mst) || (e << "s"_mst)) && // K implies s + (!(e << "z"_mst) || (e << "m"_mst)); // z implies m + assert(ok); + return e; +} + +Type ComputeType(Fragment nodetype, Type x, Type y, Type z, const std::vector<Type>& sub_types, uint32_t k, size_t data_size, size_t n_subs, size_t n_keys) { + // Sanity check on data + if (nodetype == Fragment::SHA256 || nodetype == Fragment::HASH256) { + assert(data_size == 32); + } else if (nodetype == Fragment::RIPEMD160 || nodetype == Fragment::HASH160) { + assert(data_size == 20); + } else { + assert(data_size == 0); + } + // Sanity check on k + if (nodetype == Fragment::OLDER || nodetype == Fragment::AFTER) { + assert(k >= 1 && k < 0x80000000UL); + } else if (nodetype == Fragment::MULTI) { + assert(k >= 1 && k <= n_keys); + } else if (nodetype == Fragment::THRESH) { + assert(k >= 1 && k <= n_subs); + } else { + assert(k == 0); + } + // Sanity check on subs + if (nodetype == Fragment::AND_V || nodetype == Fragment::AND_B || nodetype == Fragment::OR_B || + nodetype == Fragment::OR_C || nodetype == Fragment::OR_I || nodetype == Fragment::OR_D) { + assert(n_subs == 2); + } else if (nodetype == Fragment::ANDOR) { + assert(n_subs == 3); + } else if (nodetype == Fragment::WRAP_A || nodetype == Fragment::WRAP_S || nodetype == Fragment::WRAP_C || + nodetype == Fragment::WRAP_D || nodetype == Fragment::WRAP_V || nodetype == Fragment::WRAP_J || + nodetype == Fragment::WRAP_N) { + assert(n_subs == 1); + } else if (nodetype != Fragment::THRESH) { + assert(n_subs == 0); + } + // Sanity check on keys + if (nodetype == Fragment::PK_K || nodetype == Fragment::PK_H) { + assert(n_keys == 1); + } else if (nodetype == Fragment::MULTI) { + assert(n_keys >= 1 && n_keys <= 20); + } else { + assert(n_keys == 0); + } + + // Below is the per-nodetype logic for computing the expression types. + // It heavily relies on Type's << operator (where "X << a_mst" means + // "X has all properties listed in a"). + switch (nodetype) { + case Fragment::PK_K: return "Konudemsxk"_mst; + case Fragment::PK_H: return "Knudemsxk"_mst; + case Fragment::OLDER: return + "g"_mst.If(k & CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG) | + "h"_mst.If(!(k & CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG)) | + "Bzfmxk"_mst; + case Fragment::AFTER: return + "i"_mst.If(k >= LOCKTIME_THRESHOLD) | + "j"_mst.If(k < LOCKTIME_THRESHOLD) | + "Bzfmxk"_mst; + case Fragment::SHA256: return "Bonudmk"_mst; + case Fragment::RIPEMD160: return "Bonudmk"_mst; + case Fragment::HASH256: return "Bonudmk"_mst; + case Fragment::HASH160: return "Bonudmk"_mst; + case Fragment::JUST_1: return "Bzufmxk"_mst; + case Fragment::JUST_0: return "Bzudemsxk"_mst; + case Fragment::WRAP_A: return + "W"_mst.If(x << "B"_mst) | // W=B_x + (x & "ghijk"_mst) | // g=g_x, h=h_x, i=i_x, j=j_x, k=k_x + (x & "udfems"_mst) | // u=u_x, d=d_x, f=f_x, e=e_x, m=m_x, s=s_x + "x"_mst; // x + case Fragment::WRAP_S: return + "W"_mst.If(x << "Bo"_mst) | // W=B_x*o_x + (x & "ghijk"_mst) | // g=g_x, h=h_x, i=i_x, j=j_x, k=k_x + (x & "udfemsx"_mst); // u=u_x, d=d_x, f=f_x, e=e_x, m=m_x, s=s_x, x=x_x + case Fragment::WRAP_C: return + "B"_mst.If(x << "K"_mst) | // B=K_x + (x & "ghijk"_mst) | // g=g_x, h=h_x, i=i_x, j=j_x, k=k_x + (x & "ondfem"_mst) | // o=o_x, n=n_x, d=d_x, f=f_x, e=e_x, m=m_x + "us"_mst; // u, s + case Fragment::WRAP_D: return + "B"_mst.If(x << "Vz"_mst) | // B=V_x*z_x + "o"_mst.If(x << "z"_mst) | // o=z_x + "e"_mst.If(x << "f"_mst) | // e=f_x + (x & "ghijk"_mst) | // g=g_x, h=h_x, i=i_x, j=j_x, k=k_x + (x & "ms"_mst) | // m=m_x, s=s_x + // NOTE: 'd:' is not 'u' under P2WSH as MINIMALIF is only a policy rule there. + "ndx"_mst; // n, d, x + case Fragment::WRAP_V: return + "V"_mst.If(x << "B"_mst) | // V=B_x + (x & "ghijk"_mst) | // g=g_x, h=h_x, i=i_x, j=j_x, k=k_x + (x & "zonms"_mst) | // z=z_x, o=o_x, n=n_x, m=m_x, s=s_x + "fx"_mst; // f, x + case Fragment::WRAP_J: return + "B"_mst.If(x << "Bn"_mst) | // B=B_x*n_x + "e"_mst.If(x << "f"_mst) | // e=f_x + (x & "ghijk"_mst) | // g=g_x, h=h_x, i=i_x, j=j_x, k=k_x + (x & "oums"_mst) | // o=o_x, u=u_x, m=m_x, s=s_x + "ndx"_mst; // n, d, x + case Fragment::WRAP_N: return + (x & "ghijk"_mst) | // g=g_x, h=h_x, i=i_x, j=j_x, k=k_x + (x & "Bzondfems"_mst) | // B=B_x, z=z_x, o=o_x, n=n_x, d=d_x, f=f_x, e=e_x, m=m_x, s=s_x + "ux"_mst; // u, x + case Fragment::AND_V: return + (y & "KVB"_mst).If(x << "V"_mst) | // B=V_x*B_y, V=V_x*V_y, K=V_x*K_y + (x & "n"_mst) | (y & "n"_mst).If(x << "z"_mst) | // n=n_x+z_x*n_y + ((x | y) & "o"_mst).If((x | y) << "z"_mst) | // o=o_x*z_y+z_x*o_y + (x & y & "dmz"_mst) | // d=d_x*d_y, m=m_x*m_y, z=z_x*z_y + ((x | y) & "s"_mst) | // s=s_x+s_y + "f"_mst.If((y << "f"_mst) || (x << "s"_mst)) | // f=f_y+s_x + (y & "ux"_mst) | // u=u_y, x=x_y + ((x | y) & "ghij"_mst) | // g=g_x+g_y, h=h_x+h_y, i=i_x+i_y, j=j_x+j_y + "k"_mst.If(((x & y) << "k"_mst) && + !(((x << "g"_mst) && (y << "h"_mst)) || + ((x << "h"_mst) && (y << "g"_mst)) || + ((x << "i"_mst) && (y << "j"_mst)) || + ((x << "j"_mst) && (y << "i"_mst)))); // k=k_x*k_y*!(g_x*h_y + h_x*g_y + i_x*j_y + j_x*i_y) + case Fragment::AND_B: return + (x & "B"_mst).If(y << "W"_mst) | // B=B_x*W_y + ((x | y) & "o"_mst).If((x | y) << "z"_mst) | // o=o_x*z_y+z_x*o_y + (x & "n"_mst) | (y & "n"_mst).If(x << "z"_mst) | // n=n_x+z_x*n_y + (x & y & "e"_mst).If((x & y) << "s"_mst) | // e=e_x*e_y*s_x*s_y + (x & y & "dzm"_mst) | // d=d_x*d_y, z=z_x*z_y, m=m_x*m_y + "f"_mst.If(((x & y) << "f"_mst) || (x << "sf"_mst) || (y << "sf"_mst)) | // f=f_x*f_y + f_x*s_x + f_y*s_y + ((x | y) & "s"_mst) | // s=s_x+s_y + "ux"_mst | // u, x + ((x | y) & "ghij"_mst) | // g=g_x+g_y, h=h_x+h_y, i=i_x+i_y, j=j_x+j_y + "k"_mst.If(((x & y) << "k"_mst) && + !(((x << "g"_mst) && (y << "h"_mst)) || + ((x << "h"_mst) && (y << "g"_mst)) || + ((x << "i"_mst) && (y << "j"_mst)) || + ((x << "j"_mst) && (y << "i"_mst)))); // k=k_x*k_y*!(g_x*h_y + h_x*g_y + i_x*j_y + j_x*i_y) + case Fragment::OR_B: return + "B"_mst.If(x << "Bd"_mst && y << "Wd"_mst) | // B=B_x*d_x*W_x*d_y + ((x | y) & "o"_mst).If((x | y) << "z"_mst) | // o=o_x*z_y+z_x*o_y + (x & y & "m"_mst).If((x | y) << "s"_mst && (x & y) << "e"_mst) | // m=m_x*m_y*e_x*e_y*(s_x+s_y) + (x & y & "zse"_mst) | // z=z_x*z_y, s=s_x*s_y, e=e_x*e_y + "dux"_mst | // d, u, x + ((x | y) & "ghij"_mst) | // g=g_x+g_y, h=h_x+h_y, i=i_x+i_y, j=j_x+j_y + (x & y & "k"_mst); // k=k_x*k_y + case Fragment::OR_D: return + (y & "B"_mst).If(x << "Bdu"_mst) | // B=B_y*B_x*d_x*u_x + (x & "o"_mst).If(y << "z"_mst) | // o=o_x*z_y + (x & y & "m"_mst).If(x << "e"_mst && (x | y) << "s"_mst) | // m=m_x*m_y*e_x*(s_x+s_y) + (x & y & "zes"_mst) | // z=z_x*z_y, e=e_x*e_y, s=s_x*s_y + (y & "ufd"_mst) | // u=u_y, f=f_y, d=d_y + "x"_mst | // x + ((x | y) & "ghij"_mst) | // g=g_x+g_y, h=h_x+h_y, i=i_x+i_y, j=j_x+j_y + (x & y & "k"_mst); // k=k_x*k_y + case Fragment::OR_C: return + (y & "V"_mst).If(x << "Bdu"_mst) | // V=V_y*B_x*u_x*d_x + (x & "o"_mst).If(y << "z"_mst) | // o=o_x*z_y + (x & y & "m"_mst).If(x << "e"_mst && (x | y) << "s"_mst) | // m=m_x*m_y*e_x*(s_x+s_y) + (x & y & "zs"_mst) | // z=z_x*z_y, s=s_x*s_y + "fx"_mst | // f, x + ((x | y) & "ghij"_mst) | // g=g_x+g_y, h=h_x+h_y, i=i_x+i_y, j=j_x+j_y + (x & y & "k"_mst); // k=k_x*k_y + case Fragment::OR_I: return + (x & y & "VBKufs"_mst) | // V=V_x*V_y, B=B_x*B_y, K=K_x*K_y, u=u_x*u_y, f=f_x*f_y, s=s_x*s_y + "o"_mst.If((x & y) << "z"_mst) | // o=z_x*z_y + ((x | y) & "e"_mst).If((x | y) << "f"_mst) | // e=e_x*f_y+f_x*e_y + (x & y & "m"_mst).If((x | y) << "s"_mst) | // m=m_x*m_y*(s_x+s_y) + ((x | y) & "d"_mst) | // d=d_x+d_y + "x"_mst | // x + ((x | y) & "ghij"_mst) | // g=g_x+g_y, h=h_x+h_y, i=i_x+i_y, j=j_x+j_y + (x & y & "k"_mst); // k=k_x*k_y + case Fragment::ANDOR: return + (y & z & "BKV"_mst).If(x << "Bdu"_mst) | // B=B_x*d_x*u_x*B_y*B_z, K=B_x*d_x*u_x*K_y*K_z, V=B_x*d_x*u_x*V_y*V_z + (x & y & z & "z"_mst) | // z=z_x*z_y*z_z + ((x | (y & z)) & "o"_mst).If((x | (y & z)) << "z"_mst) | // o=o_x*z_y*z_z+z_x*o_y*o_z + (y & z & "u"_mst) | // u=u_y*u_z + (z & "f"_mst).If((x << "s"_mst) || (y << "f"_mst)) | // f=(s_x+f_y)*f_z + (z & "d"_mst) | // d=d_z + (x & z & "e"_mst).If(x << "s"_mst || y << "f"_mst) | // e=e_x*e_z*(s_x+f_y) + (x & y & z & "m"_mst).If(x << "e"_mst && (x | y | z) << "s"_mst) | // m=m_x*m_y*m_z*e_x*(s_x+s_y+s_z) + (z & (x | y) & "s"_mst) | // s=s_z*(s_x+s_y) + "x"_mst | // x + ((x | y | z) & "ghij"_mst) | // g=g_x+g_y+g_z, h=h_x+h_y+h_z, i=i_x+i_y+i_z, j=j_x+j_y_j_z + "k"_mst.If(((x & y & z) << "k"_mst) && + !(((x << "g"_mst) && (y << "h"_mst)) || + ((x << "h"_mst) && (y << "g"_mst)) || + ((x << "i"_mst) && (y << "j"_mst)) || + ((x << "j"_mst) && (y << "i"_mst)))); // k=k_x*k_y*k_z* !(g_x*h_y + h_x*g_y + i_x*j_y + j_x*i_y) + case Fragment::MULTI: return "Bnudemsk"_mst; + case Fragment::THRESH: { + bool all_e = true; + bool all_m = true; + uint32_t args = 0; + uint32_t num_s = 0; + Type acc_tl = "k"_mst; + for (size_t i = 0; i < sub_types.size(); ++i) { + Type t = sub_types[i]; + if (!(t << (i ? "Wdu"_mst : "Bdu"_mst))) return ""_mst; // Require Bdu, Wdu, Wdu, ... + if (!(t << "e"_mst)) all_e = false; + if (!(t << "m"_mst)) all_m = false; + if (t << "s"_mst) num_s += 1; + args += (t << "z"_mst) ? 0 : (t << "o"_mst) ? 1 : 2; + acc_tl = ((acc_tl | t) & "ghij"_mst) | + // Thresh contains a combination of timelocks if it has threshold > 1 and + // it contains two different children that have different types of timelocks + // Note how if any of the children don't have "k", the parent also does not have "k" + "k"_mst.If(((acc_tl & t) << "k"_mst) && ((k <= 1) || + ((k > 1) && !(((acc_tl << "g"_mst) && (t << "h"_mst)) || + ((acc_tl << "h"_mst) && (t << "g"_mst)) || + ((acc_tl << "i"_mst) && (t << "j"_mst)) || + ((acc_tl << "j"_mst) && (t << "i"_mst)))))); + } + return "Bdu"_mst | + "z"_mst.If(args == 0) | // z=all z + "o"_mst.If(args == 1) | // o=all z except one o + "e"_mst.If(all_e && num_s == n_subs) | // e=all e and all s + "m"_mst.If(all_e && all_m && num_s >= n_subs - k) | // m=all e, >=(n-k) s + "s"_mst.If(num_s >= n_subs - k + 1) | // s= >=(n-k+1) s + acc_tl; // timelock info + } + } + assert(false); + return ""_mst; +} + +size_t ComputeScriptLen(Fragment nodetype, Type sub0typ, size_t subsize, uint32_t k, size_t n_subs, size_t n_keys) { + switch (nodetype) { + case Fragment::JUST_1: + case Fragment::JUST_0: return 1; + case Fragment::PK_K: return 34; + case Fragment::PK_H: return 3 + 21; + case Fragment::OLDER: + case Fragment::AFTER: return 1 + BuildScript(k).size(); + case Fragment::HASH256: + case Fragment::SHA256: return 4 + 2 + 33; + case Fragment::HASH160: + case Fragment::RIPEMD160: return 4 + 2 + 21; + case Fragment::MULTI: return 3 + (n_keys > 16) + (k > 16) + 34 * n_keys; + case Fragment::AND_V: return subsize; + case Fragment::WRAP_V: return subsize + (sub0typ << "x"_mst); + case Fragment::WRAP_S: + case Fragment::WRAP_C: + case Fragment::WRAP_N: + case Fragment::AND_B: + case Fragment::OR_B: return subsize + 1; + case Fragment::WRAP_A: + case Fragment::OR_C: return subsize + 2; + case Fragment::WRAP_D: + case Fragment::OR_D: + case Fragment::OR_I: + case Fragment::ANDOR: return subsize + 3; + case Fragment::WRAP_J: return subsize + 4; + case Fragment::THRESH: return subsize + n_subs + BuildScript(k).size(); + } + assert(false); + return 0; +} + +bool DecomposeScript(const CScript& script, std::vector<std::pair<opcodetype, std::vector<unsigned char>>>& out) +{ + out.clear(); + CScript::const_iterator it = script.begin(), itend = script.end(); + while (it != itend) { + std::vector<unsigned char> push_data; + opcodetype opcode; + if (!script.GetOp(it, opcode, push_data)) { + out.clear(); + return false; + } else if (opcode >= OP_1 && opcode <= OP_16) { + // Deal with OP_n (GetOp does not turn them into pushes). + push_data.assign(1, CScript::DecodeOP_N(opcode)); + } else if (opcode == OP_CHECKSIGVERIFY) { + // Decompose OP_CHECKSIGVERIFY into OP_CHECKSIG OP_VERIFY + out.emplace_back(OP_CHECKSIG, std::vector<unsigned char>()); + opcode = OP_VERIFY; + } else if (opcode == OP_CHECKMULTISIGVERIFY) { + // Decompose OP_CHECKMULTISIGVERIFY into OP_CHECKMULTISIG OP_VERIFY + out.emplace_back(OP_CHECKMULTISIG, std::vector<unsigned char>()); + opcode = OP_VERIFY; + } else if (opcode == OP_EQUALVERIFY) { + // Decompose OP_EQUALVERIFY into OP_EQUAL OP_VERIFY + out.emplace_back(OP_EQUAL, std::vector<unsigned char>()); + opcode = OP_VERIFY; + } else if (IsPushdataOp(opcode)) { + if (!CheckMinimalPush(push_data, opcode)) return false; + } else if (it != itend && (opcode == OP_CHECKSIG || opcode == OP_CHECKMULTISIG || opcode == OP_EQUAL) && (*it == OP_VERIFY)) { + // Rule out non minimal VERIFY sequences + return false; + } + out.emplace_back(opcode, std::move(push_data)); + } + std::reverse(out.begin(), out.end()); + return true; +} + +bool ParseScriptNumber(const std::pair<opcodetype, std::vector<unsigned char>>& in, int64_t& k) { + if (in.first == OP_0) { + k = 0; + return true; + } + if (!in.second.empty()) { + if (IsPushdataOp(in.first) && !CheckMinimalPush(in.second, in.first)) return false; + try { + k = CScriptNum(in.second, true).GetInt64(); + return true; + } catch(const scriptnum_error&) {} + } + return false; +} + +int FindNextChar(Span<const char> sp, const char m) +{ + for (int i = 0; i < (int)sp.size(); ++i) { + if (sp[i] == m) return i; + // We only search within the current parentheses + if (sp[i] == ')') break; + } + return -1; +} + +} // namespace internal +} // namespace miniscript diff --git a/src/script/miniscript.h b/src/script/miniscript.h new file mode 100644 index 0000000000..5c1cc316dc --- /dev/null +++ b/src/script/miniscript.h @@ -0,0 +1,1652 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_SCRIPT_MINISCRIPT_H +#define BITCOIN_SCRIPT_MINISCRIPT_H + +#include <algorithm> +#include <numeric> +#include <memory> +#include <optional> +#include <string> +#include <vector> + +#include <stdlib.h> +#include <assert.h> + +#include <policy/policy.h> +#include <primitives/transaction.h> +#include <script/script.h> +#include <span.h> +#include <util/spanparsing.h> +#include <util/strencodings.h> +#include <util/string.h> +#include <util/vector.h> + +namespace miniscript { + +/** This type encapsulates the miniscript type system properties. + * + * Every miniscript expression is one of 4 basic types, and additionally has + * a number of boolean type properties. + * + * The basic types are: + * - "B" Base: + * - Takes its inputs from the top of the stack. + * - When satisfied, pushes a nonzero value of up to 4 bytes onto the stack. + * - When dissatisfied, pushes a 0 onto the stack. + * - This is used for most expressions, and required for the top level one. + * - For example: older(n) = <n> OP_CHECKSEQUENCEVERIFY. + * - "V" Verify: + * - Takes its inputs from the top of the stack. + * - When satisfactied, pushes nothing. + * - Cannot be dissatisfied. + * - This can be obtained by adding an OP_VERIFY to a B, modifying the last opcode + * of a B to its -VERIFY version (only for OP_CHECKSIG, OP_CHECKSIGVERIFY + * and OP_EQUAL), or by combining a V fragment under some conditions. + * - For example vc:pk_k(key) = <key> OP_CHECKSIGVERIFY + * - "K" Key: + * - Takes its inputs from the top of the stack. + * - Becomes a B when followed by OP_CHECKSIG. + * - Always pushes a public key onto the stack, for which a signature is to be + * provided to satisfy the expression. + * - For example pk_h(key) = OP_DUP OP_HASH160 <Hash160(key)> OP_EQUALVERIFY + * - "W" Wrapped: + * - Takes its input from one below the top of the stack. + * - When satisfied, pushes a nonzero value (like B) on top of the stack, or one below. + * - When dissatisfied, pushes 0 op top of the stack or one below. + * - Is always "OP_SWAP [B]" or "OP_TOALTSTACK [B] OP_FROMALTSTACK". + * - For example sc:pk_k(key) = OP_SWAP <key> OP_CHECKSIG + * + * There a type properties that help reasoning about correctness: + * - "z" Zero-arg: + * - Is known to always consume exactly 0 stack elements. + * - For example after(n) = <n> OP_CHECKLOCKTIMEVERIFY + * - "o" One-arg: + * - Is known to always consume exactly 1 stack element. + * - Conflicts with property 'z' + * - For example sha256(hash) = OP_SIZE 32 OP_EQUALVERIFY OP_SHA256 <hash> OP_EQUAL + * - "n" Nonzero: + * - For every way this expression can be satisfied, a satisfaction exists that never needs + * a zero top stack element. + * - Conflicts with property 'z' and with type 'W'. + * - "d" Dissatisfiable: + * - There is an easy way to construct a dissatisfaction for this expression. + * - Conflicts with type 'V'. + * - "u" Unit: + * - In case of satisfaction, an exact 1 is put on the stack (rather than just nonzero). + * - Conflicts with type 'V'. + * + * Additional type properties help reasoning about nonmalleability: + * - "e" Expression: + * - This implies property 'd', but the dissatisfaction is nonmalleable. + * - This generally requires 'e' for all subexpressions which are invoked for that + * dissatifsaction, and property 'f' for the unexecuted subexpressions in that case. + * - Conflicts with type 'V'. + * - "f" Forced: + * - Dissatisfactions (if any) for this expression always involve at least one signature. + * - Is always true for type 'V'. + * - "s" Safe: + * - Satisfactions for this expression always involve at least one signature. + * - "m" Nonmalleable: + * - For every way this expression can be satisfied (which may be none), + * a nonmalleable satisfaction exists. + * - This generally requires 'm' for all subexpressions, and 'e' for all subexpressions + * which are dissatisfied when satisfying the parent. + * + * One type property is an implementation detail: + * - "x" Expensive verify: + * - Expressions with this property have a script whose last opcode is not EQUAL, CHECKSIG, or CHECKMULTISIG. + * - Not having this property means that it can be converted to a V at no cost (by switching to the + * -VERIFY version of the last opcode). + * + * Five more type properties for representing timelock information. Spend paths + * in miniscripts containing conflicting timelocks and heightlocks cannot be spent together. + * This helps users detect if miniscript does not match the semantic behaviour the + * user expects. + * - "g" Whether the branch contains a relative time timelock + * - "h" Whether the branch contains a relative height timelock + * - "i" Whether the branch contains an absolute time timelock + * - "j" Whether the branch contains an absolute height timelock + * - "k" + * - Whether all satisfactions of this expression don't contain a mix of heightlock and timelock + * of the same type. + * - If the miniscript does not have the "k" property, the miniscript template will not match + * the user expectation of the corresponding spending policy. + * For each of these properties the subset rule holds: an expression with properties X, Y, and Z, is also + * valid in places where an X, a Y, a Z, an XY, ... is expected. +*/ +class Type { + //! Internal bitmap of properties (see ""_mst operator for details). + uint32_t m_flags; + + //! Internal constructor used by the ""_mst operator. + explicit constexpr Type(uint32_t flags) : m_flags(flags) {} + +public: + //! The only way to publicly construct a Type is using this literal operator. + friend constexpr Type operator"" _mst(const char* c, size_t l); + + //! Compute the type with the union of properties. + constexpr Type operator|(Type x) const { return Type(m_flags | x.m_flags); } + + //! Compute the type with the intersection of properties. + constexpr Type operator&(Type x) const { return Type(m_flags & x.m_flags); } + + //! Check whether the left hand's properties are superset of the right's (= left is a subtype of right). + constexpr bool operator<<(Type x) const { return (x.m_flags & ~m_flags) == 0; } + + //! Comparison operator to enable use in sets/maps (total ordering incompatible with <<). + constexpr bool operator<(Type x) const { return m_flags < x.m_flags; } + + //! Equality operator. + constexpr bool operator==(Type x) const { return m_flags == x.m_flags; } + + //! The empty type if x is false, itself otherwise. + constexpr Type If(bool x) const { return Type(x ? m_flags : 0); } +}; + +//! Literal operator to construct Type objects. +inline constexpr Type operator"" _mst(const char* c, size_t l) { + Type typ{0}; + + for (const char *p = c; p < c + l; p++) { + typ = typ | Type( + *p == 'B' ? 1 << 0 : // Base type + *p == 'V' ? 1 << 1 : // Verify type + *p == 'K' ? 1 << 2 : // Key type + *p == 'W' ? 1 << 3 : // Wrapped type + *p == 'z' ? 1 << 4 : // Zero-arg property + *p == 'o' ? 1 << 5 : // One-arg property + *p == 'n' ? 1 << 6 : // Nonzero arg property + *p == 'd' ? 1 << 7 : // Dissatisfiable property + *p == 'u' ? 1 << 8 : // Unit property + *p == 'e' ? 1 << 9 : // Expression property + *p == 'f' ? 1 << 10 : // Forced property + *p == 's' ? 1 << 11 : // Safe property + *p == 'm' ? 1 << 12 : // Nonmalleable property + *p == 'x' ? 1 << 13 : // Expensive verify + *p == 'g' ? 1 << 14 : // older: contains relative time timelock (csv_time) + *p == 'h' ? 1 << 15 : // older: contains relative height timelock (csv_height) + *p == 'i' ? 1 << 16 : // after: contains time timelock (cltv_time) + *p == 'j' ? 1 << 17 : // after: contains height timelock (cltv_height) + *p == 'k' ? 1 << 18 : // does not contain a combination of height and time locks + (throw std::logic_error("Unknown character in _mst literal"), 0) + ); + } + + return typ; +} + +template<typename Key> struct Node; +template<typename Key> using NodeRef = std::shared_ptr<const Node<Key>>; + +//! Construct a miniscript node as a shared_ptr. +template<typename Key, typename... Args> +NodeRef<Key> MakeNodeRef(Args&&... args) { return std::make_shared<const Node<Key>>(std::forward<Args>(args)...); } + +//! The different node types in miniscript. +enum class Fragment { + JUST_0, //!< OP_0 + JUST_1, //!< OP_1 + PK_K, //!< [key] + PK_H, //!< OP_DUP OP_HASH160 [keyhash] OP_EQUALVERIFY + OLDER, //!< [n] OP_CHECKSEQUENCEVERIFY + AFTER, //!< [n] OP_CHECKLOCKTIMEVERIFY + SHA256, //!< OP_SIZE 32 OP_EQUALVERIFY OP_SHA256 [hash] OP_EQUAL + HASH256, //!< OP_SIZE 32 OP_EQUALVERIFY OP_HASH256 [hash] OP_EQUAL + RIPEMD160, //!< OP_SIZE 32 OP_EQUALVERIFY OP_RIPEMD160 [hash] OP_EQUAL + HASH160, //!< OP_SIZE 32 OP_EQUALVERIFY OP_HASH160 [hash] OP_EQUAL + WRAP_A, //!< OP_TOALTSTACK [X] OP_FROMALTSTACK + WRAP_S, //!< OP_SWAP [X] + WRAP_C, //!< [X] OP_CHECKSIG + WRAP_D, //!< OP_DUP OP_IF [X] OP_ENDIF + WRAP_V, //!< [X] OP_VERIFY (or -VERIFY version of last opcode in X) + WRAP_J, //!< OP_SIZE OP_0NOTEQUAL OP_IF [X] OP_ENDIF + WRAP_N, //!< [X] OP_0NOTEQUAL + AND_V, //!< [X] [Y] + AND_B, //!< [X] [Y] OP_BOOLAND + OR_B, //!< [X] [Y] OP_BOOLOR + OR_C, //!< [X] OP_NOTIF [Y] OP_ENDIF + OR_D, //!< [X] OP_IFDUP OP_NOTIF [Y] OP_ENDIF + OR_I, //!< OP_IF [X] OP_ELSE [Y] OP_ENDIF + ANDOR, //!< [X] OP_NOTIF [Z] OP_ELSE [Y] OP_ENDIF + THRESH, //!< [X1] ([Xn] OP_ADD)* [k] OP_EQUAL + MULTI, //!< [k] [key_n]* [n] OP_CHECKMULTISIG + // AND_N(X,Y) is represented as ANDOR(X,Y,0) + // WRAP_T(X) is represented as AND_V(X,1) + // WRAP_L(X) is represented as OR_I(0,X) + // WRAP_U(X) is represented as OR_I(X,0) +}; + + +namespace internal { + +//! Helper function for Node::CalcType. +Type ComputeType(Fragment nodetype, Type x, Type y, Type z, const std::vector<Type>& sub_types, uint32_t k, size_t data_size, size_t n_subs, size_t n_keys); + +//! Helper function for Node::CalcScriptLen. +size_t ComputeScriptLen(Fragment nodetype, Type sub0typ, size_t subsize, uint32_t k, size_t n_subs, size_t n_keys); + +//! A helper sanitizer/checker for the output of CalcType. +Type SanitizeType(Type x); + +//! Class whose objects represent the maximum of a list of integers. +template<typename I> +struct MaxInt { + const bool valid; + const I value; + + MaxInt() : valid(false), value(0) {} + MaxInt(I val) : valid(true), value(val) {} + + friend MaxInt<I> operator+(const MaxInt<I>& a, const MaxInt<I>& b) { + if (!a.valid || !b.valid) return {}; + return a.value + b.value; + } + + friend MaxInt<I> operator|(const MaxInt<I>& a, const MaxInt<I>& b) { + if (!a.valid) return b; + if (!b.valid) return a; + return std::max(a.value, b.value); + } +}; + +struct Ops { + //! Non-push opcodes. + uint32_t count; + //! Number of keys in possibly executed OP_CHECKMULTISIG(VERIFY)s to satisfy. + MaxInt<uint32_t> sat; + //! Number of keys in possibly executed OP_CHECKMULTISIG(VERIFY)s to dissatisfy. + MaxInt<uint32_t> dsat; + + Ops(uint32_t in_count, MaxInt<uint32_t> in_sat, MaxInt<uint32_t> in_dsat) : count(in_count), sat(in_sat), dsat(in_dsat) {}; +}; + +struct StackSize { + //! Maximum stack size to satisfy; + MaxInt<uint32_t> sat; + //! Maximum stack size to dissatisfy; + MaxInt<uint32_t> dsat; + + StackSize(MaxInt<uint32_t> in_sat, MaxInt<uint32_t> in_dsat) : sat(in_sat), dsat(in_dsat) {}; +}; + +} // namespace internal + +//! A node in a miniscript expression. +template<typename Key> +struct Node { + //! What node type this node is. + const Fragment nodetype; + //! The k parameter (time for OLDER/AFTER, threshold for THRESH(_M)) + const uint32_t k = 0; + //! The keys used by this expression (only for PK_K/PK_H/MULTI) + const std::vector<Key> keys; + //! The data bytes in this expression (only for HASH160/HASH256/SHA256/RIPEMD10). + const std::vector<unsigned char> data; + //! Subexpressions (for WRAP_*/AND_*/OR_*/ANDOR/THRESH) + const std::vector<NodeRef<Key>> subs; + +private: + //! Cached ops counts. + const internal::Ops ops; + //! Cached stack size bounds. + const internal::StackSize ss; + //! Cached expression type (computed by CalcType and fed through SanitizeType). + const Type typ; + //! Cached script length (computed by CalcScriptLen). + const size_t scriptlen; + + //! Compute the length of the script for this miniscript (including children). + size_t CalcScriptLen() const { + size_t subsize = 0; + for (const auto& sub : subs) { + subsize += sub->ScriptSize(); + } + Type sub0type = subs.size() > 0 ? subs[0]->GetType() : ""_mst; + return internal::ComputeScriptLen(nodetype, sub0type, subsize, k, subs.size(), keys.size()); + } + + /* Apply a recursive algorithm to a Miniscript tree, without actual recursive calls. + * + * The algorithm is defined by two functions: downfn and upfn. Conceptually, the + * result can be thought of as first using downfn to compute a "state" for each node, + * from the root down to the leaves. Then upfn is used to compute a "result" for each + * node, from the leaves back up to the root, which is then returned. In the actual + * implementation, both functions are invoked in an interleaved fashion, performing a + * depth-first traversal of the tree. + * + * In more detail, it is invoked as node.TreeEvalMaybe<Result>(root, downfn, upfn): + * - root is the state of the root node, of type State. + * - downfn is a callable (State&, const Node&, size_t) -> State, which given a + * node, its state, and an index of one of its children, computes the state of that + * child. It can modify the state. Children of a given node will have downfn() + * called in order. + * - upfn is a callable (State&&, const Node&, Span<Result>) -> std::optional<Result>, + * which given a node, its state, and a Span of the results of its children, + * computes the result of the node. If std::nullopt is returned by upfn, + * TreeEvalMaybe() immediately returns std::nullopt. + * The return value of TreeEvalMaybe is the result of the root node. + */ + template<typename Result, typename State, typename DownFn, typename UpFn> + std::optional<Result> TreeEvalMaybe(State root_state, DownFn downfn, UpFn upfn) const + { + /** Entries of the explicit stack tracked in this algorithm. */ + struct StackElem + { + const Node& node; //!< The node being evaluated. + size_t expanded; //!< How many children of this node have been expanded. + State state; //!< The state for that node. + + StackElem(const Node& node_, size_t exp_, State&& state_) : + node(node_), expanded(exp_), state(std::move(state_)) {} + }; + /* Stack of tree nodes being explored. */ + std::vector<StackElem> stack; + /* Results of subtrees so far. Their order and mapping to tree nodes + * is implicitly defined by stack. */ + std::vector<Result> results; + stack.emplace_back(*this, 0, std::move(root_state)); + + /* Here is a demonstration of the algorithm, for an example tree A(B,C(D,E),F). + * State variables are omitted for simplicity. + * + * First: stack=[(A,0)] results=[] + * stack=[(A,1),(B,0)] results=[] + * stack=[(A,1)] results=[B] + * stack=[(A,2),(C,0)] results=[B] + * stack=[(A,2),(C,1),(D,0)] results=[B] + * stack=[(A,2),(C,1)] results=[B,D] + * stack=[(A,2),(C,2),(E,0)] results=[B,D] + * stack=[(A,2),(C,2)] results=[B,D,E] + * stack=[(A,2)] results=[B,C] + * stack=[(A,3),(F,0)] results=[B,C] + * stack=[(A,3)] results=[B,C,F] + * Final: stack=[] results=[A] + */ + while (stack.size()) { + const Node& node = stack.back().node; + if (stack.back().expanded < node.subs.size()) { + /* We encounter a tree node with at least one unexpanded child. + * Expand it. By the time we hit this node again, the result of + * that child (and all earlier children) will be at the end of `results`. */ + size_t child_index = stack.back().expanded++; + State child_state = downfn(stack.back().state, node, child_index); + stack.emplace_back(*node.subs[child_index], 0, std::move(child_state)); + continue; + } + // Invoke upfn with the last node.subs.size() elements of results as input. + assert(results.size() >= node.subs.size()); + std::optional<Result> result{upfn(std::move(stack.back().state), node, + Span<Result>{results}.last(node.subs.size()))}; + // If evaluation returns std::nullopt, abort immediately. + if (!result) return {}; + // Replace the last node.subs.size() elements of results with the new result. + results.erase(results.end() - node.subs.size(), results.end()); + results.push_back(std::move(*result)); + stack.pop_back(); + } + // The final remaining results element is the root result, return it. + assert(results.size() == 1); + return std::move(results[0]); + } + + /** Like TreeEvalMaybe, but always produces a result. upfn must return Result. */ + template<typename Result, typename State, typename DownFn, typename UpFn> + Result TreeEval(State root_state, DownFn&& downfn, UpFn upfn) const + { + // Invoke TreeEvalMaybe with upfn wrapped to return std::optional<Result>, and then + // unconditionally dereference the result (it cannot be std::nullopt). + return std::move(*TreeEvalMaybe<Result>(std::move(root_state), + std::forward<DownFn>(downfn), + [&upfn](State&& state, const Node& node, Span<Result> subs) { + Result res{upfn(std::move(state), node, subs)}; + return std::optional<Result>(std::move(res)); + } + )); + } + + //! Compute the type for this miniscript. + Type CalcType() const { + using namespace internal; + + // THRESH has a variable number of subexpressions + std::vector<Type> sub_types; + if (nodetype == Fragment::THRESH) { + for (const auto& sub : subs) sub_types.push_back(sub->GetType()); + } + // All other nodes than THRESH can be computed just from the types of the 0-3 subexpressions. + Type x = subs.size() > 0 ? subs[0]->GetType() : ""_mst; + Type y = subs.size() > 1 ? subs[1]->GetType() : ""_mst; + Type z = subs.size() > 2 ? subs[2]->GetType() : ""_mst; + + return SanitizeType(ComputeType(nodetype, x, y, z, sub_types, k, data.size(), subs.size(), keys.size())); + } + +public: + template<typename Ctx> + CScript ToScript(const Ctx& ctx) const + { + // To construct the CScript for a Miniscript object, we use the TreeEval algorithm. + // The State is a boolean: whether or not the node's script expansion is followed + // by an OP_VERIFY (which may need to be combined with the last script opcode). + auto downfn = [](bool verify, const Node& node, size_t index) { + // For WRAP_V, the subexpression is certainly followed by OP_VERIFY. + if (node.nodetype == Fragment::WRAP_V) return true; + // The subexpression of WRAP_S, and the last subexpression of AND_V + // inherit the followed-by-OP_VERIFY property from the parent. + if (node.nodetype == Fragment::WRAP_S || + (node.nodetype == Fragment::AND_V && index == 1)) return verify; + return false; + }; + // The upward function computes for a node, given its followed-by-OP_VERIFY status + // and the CScripts of its child nodes, the CScript of the node. + auto upfn = [&ctx](bool verify, const Node& node, Span<CScript> subs) -> CScript { + switch (node.nodetype) { + case Fragment::PK_K: return BuildScript(ctx.ToPKBytes(node.keys[0])); + case Fragment::PK_H: return BuildScript(OP_DUP, OP_HASH160, ctx.ToPKHBytes(node.keys[0]), OP_EQUALVERIFY); + case Fragment::OLDER: return BuildScript(node.k, OP_CHECKSEQUENCEVERIFY); + case Fragment::AFTER: return BuildScript(node.k, OP_CHECKLOCKTIMEVERIFY); + case Fragment::SHA256: return BuildScript(OP_SIZE, 32, OP_EQUALVERIFY, OP_SHA256, node.data, verify ? OP_EQUALVERIFY : OP_EQUAL); + case Fragment::RIPEMD160: return BuildScript(OP_SIZE, 32, OP_EQUALVERIFY, OP_RIPEMD160, node.data, verify ? OP_EQUALVERIFY : OP_EQUAL); + case Fragment::HASH256: return BuildScript(OP_SIZE, 32, OP_EQUALVERIFY, OP_HASH256, node.data, verify ? OP_EQUALVERIFY : OP_EQUAL); + case Fragment::HASH160: return BuildScript(OP_SIZE, 32, OP_EQUALVERIFY, OP_HASH160, node.data, verify ? OP_EQUALVERIFY : OP_EQUAL); + case Fragment::WRAP_A: return BuildScript(OP_TOALTSTACK, subs[0], OP_FROMALTSTACK); + case Fragment::WRAP_S: return BuildScript(OP_SWAP, subs[0]); + case Fragment::WRAP_C: return BuildScript(std::move(subs[0]), verify ? OP_CHECKSIGVERIFY : OP_CHECKSIG); + case Fragment::WRAP_D: return BuildScript(OP_DUP, OP_IF, subs[0], OP_ENDIF); + case Fragment::WRAP_V: { + if (node.subs[0]->GetType() << "x"_mst) { + return BuildScript(std::move(subs[0]), OP_VERIFY); + } else { + return std::move(subs[0]); + } + } + case Fragment::WRAP_J: return BuildScript(OP_SIZE, OP_0NOTEQUAL, OP_IF, subs[0], OP_ENDIF); + case Fragment::WRAP_N: return BuildScript(std::move(subs[0]), OP_0NOTEQUAL); + case Fragment::JUST_1: return BuildScript(OP_1); + case Fragment::JUST_0: return BuildScript(OP_0); + case Fragment::AND_V: return BuildScript(std::move(subs[0]), subs[1]); + case Fragment::AND_B: return BuildScript(std::move(subs[0]), subs[1], OP_BOOLAND); + case Fragment::OR_B: return BuildScript(std::move(subs[0]), subs[1], OP_BOOLOR); + case Fragment::OR_D: return BuildScript(std::move(subs[0]), OP_IFDUP, OP_NOTIF, subs[1], OP_ENDIF); + case Fragment::OR_C: return BuildScript(std::move(subs[0]), OP_NOTIF, subs[1], OP_ENDIF); + case Fragment::OR_I: return BuildScript(OP_IF, subs[0], OP_ELSE, subs[1], OP_ENDIF); + case Fragment::ANDOR: return BuildScript(std::move(subs[0]), OP_NOTIF, subs[2], OP_ELSE, subs[1], OP_ENDIF); + case Fragment::MULTI: { + CScript script = BuildScript(node.k); + for (const auto& key : node.keys) { + script = BuildScript(std::move(script), ctx.ToPKBytes(key)); + } + return BuildScript(std::move(script), node.keys.size(), verify ? OP_CHECKMULTISIGVERIFY : OP_CHECKMULTISIG); + } + case Fragment::THRESH: { + CScript script = std::move(subs[0]); + for (size_t i = 1; i < subs.size(); ++i) { + script = BuildScript(std::move(script), subs[i], OP_ADD); + } + return BuildScript(std::move(script), node.k, verify ? OP_EQUALVERIFY : OP_EQUAL); + } + } + assert(false); + return {}; + }; + return TreeEval<CScript>(false, downfn, upfn); + } + + template<typename CTx> + bool ToString(const CTx& ctx, std::string& ret) const { + // To construct the std::string representation for a Miniscript object, we use + // the TreeEvalMaybe algorithm. The State is a boolean: whether the parent node is a + // wrapper. If so, non-wrapper expressions must be prefixed with a ":". + auto downfn = [](bool, const Node& node, size_t) { + return (node.nodetype == Fragment::WRAP_A || node.nodetype == Fragment::WRAP_S || + node.nodetype == Fragment::WRAP_D || node.nodetype == Fragment::WRAP_V || + node.nodetype == Fragment::WRAP_J || node.nodetype == Fragment::WRAP_N || + node.nodetype == Fragment::WRAP_C || + (node.nodetype == Fragment::AND_V && node.subs[1]->nodetype == Fragment::JUST_1) || + (node.nodetype == Fragment::OR_I && node.subs[0]->nodetype == Fragment::JUST_0) || + (node.nodetype == Fragment::OR_I && node.subs[1]->nodetype == Fragment::JUST_0)); + }; + // The upward function computes for a node, given whether its parent is a wrapper, + // and the string representations of its child nodes, the string representation of the node. + auto upfn = [&ctx](bool wrapped, const Node& node, Span<std::string> subs) -> std::optional<std::string> { + std::string ret = wrapped ? ":" : ""; + + switch (node.nodetype) { + case Fragment::WRAP_A: return "a" + std::move(subs[0]); + case Fragment::WRAP_S: return "s" + std::move(subs[0]); + case Fragment::WRAP_C: + if (node.subs[0]->nodetype == Fragment::PK_K) { + // pk(K) is syntactic sugar for c:pk_k(K) + std::string key_str; + if (!ctx.ToString(node.subs[0]->keys[0], key_str)) return {}; + return std::move(ret) + "pk(" + std::move(key_str) + ")"; + } + if (node.subs[0]->nodetype == Fragment::PK_H) { + // pkh(K) is syntactic sugar for c:pk_h(K) + std::string key_str; + if (!ctx.ToString(node.subs[0]->keys[0], key_str)) return {}; + return std::move(ret) + "pkh(" + std::move(key_str) + ")"; + } + return "c" + std::move(subs[0]); + case Fragment::WRAP_D: return "d" + std::move(subs[0]); + case Fragment::WRAP_V: return "v" + std::move(subs[0]); + case Fragment::WRAP_J: return "j" + std::move(subs[0]); + case Fragment::WRAP_N: return "n" + std::move(subs[0]); + case Fragment::AND_V: + // t:X is syntactic sugar for and_v(X,1). + if (node.subs[1]->nodetype == Fragment::JUST_1) return "t" + std::move(subs[0]); + break; + case Fragment::OR_I: + if (node.subs[0]->nodetype == Fragment::JUST_0) return "l" + std::move(subs[1]); + if (node.subs[1]->nodetype == Fragment::JUST_0) return "u" + std::move(subs[0]); + break; + default: break; + } + switch (node.nodetype) { + case Fragment::PK_K: { + std::string key_str; + if (!ctx.ToString(node.keys[0], key_str)) return {}; + return std::move(ret) + "pk_k(" + std::move(key_str) + ")"; + } + case Fragment::PK_H: { + std::string key_str; + if (!ctx.ToString(node.keys[0], key_str)) return {}; + return std::move(ret) + "pk_h(" + std::move(key_str) + ")"; + } + case Fragment::AFTER: return std::move(ret) + "after(" + ::ToString(node.k) + ")"; + case Fragment::OLDER: return std::move(ret) + "older(" + ::ToString(node.k) + ")"; + case Fragment::HASH256: return std::move(ret) + "hash256(" + HexStr(node.data) + ")"; + case Fragment::HASH160: return std::move(ret) + "hash160(" + HexStr(node.data) + ")"; + case Fragment::SHA256: return std::move(ret) + "sha256(" + HexStr(node.data) + ")"; + case Fragment::RIPEMD160: return std::move(ret) + "ripemd160(" + HexStr(node.data) + ")"; + case Fragment::JUST_1: return std::move(ret) + "1"; + case Fragment::JUST_0: return std::move(ret) + "0"; + case Fragment::AND_V: return std::move(ret) + "and_v(" + std::move(subs[0]) + "," + std::move(subs[1]) + ")"; + case Fragment::AND_B: return std::move(ret) + "and_b(" + std::move(subs[0]) + "," + std::move(subs[1]) + ")"; + case Fragment::OR_B: return std::move(ret) + "or_b(" + std::move(subs[0]) + "," + std::move(subs[1]) + ")"; + case Fragment::OR_D: return std::move(ret) + "or_d(" + std::move(subs[0]) + "," + std::move(subs[1]) + ")"; + case Fragment::OR_C: return std::move(ret) + "or_c(" + std::move(subs[0]) + "," + std::move(subs[1]) + ")"; + case Fragment::OR_I: return std::move(ret) + "or_i(" + std::move(subs[0]) + "," + std::move(subs[1]) + ")"; + case Fragment::ANDOR: + // and_n(X,Y) is syntactic sugar for andor(X,Y,0). + if (node.subs[2]->nodetype == Fragment::JUST_0) return std::move(ret) + "and_n(" + std::move(subs[0]) + "," + std::move(subs[1]) + ")"; + return std::move(ret) + "andor(" + std::move(subs[0]) + "," + std::move(subs[1]) + "," + std::move(subs[2]) + ")"; + case Fragment::MULTI: { + auto str = std::move(ret) + "multi(" + ::ToString(node.k); + for (const auto& key : node.keys) { + std::string key_str; + if (!ctx.ToString(key, key_str)) return {}; + str += "," + std::move(key_str); + } + return std::move(str) + ")"; + } + case Fragment::THRESH: { + auto str = std::move(ret) + "thresh(" + ::ToString(node.k); + for (auto& sub : subs) { + str += "," + std::move(sub); + } + return std::move(str) + ")"; + } + default: assert(false); + } + return ""; // Should never be reached. + }; + + auto res = TreeEvalMaybe<std::string>(false, downfn, upfn); + if (res.has_value()) ret = std::move(*res); + return res.has_value(); + } + + internal::Ops CalcOps() const { + switch (nodetype) { + case Fragment::JUST_1: return {0, 0, {}}; + case Fragment::JUST_0: return {0, {}, 0}; + case Fragment::PK_K: return {0, 0, 0}; + case Fragment::PK_H: return {3, 0, 0}; + case Fragment::OLDER: + case Fragment::AFTER: return {1, 0, {}}; + case Fragment::SHA256: + case Fragment::RIPEMD160: + case Fragment::HASH256: + case Fragment::HASH160: return {4, 0, {}}; + case Fragment::AND_V: return {subs[0]->ops.count + subs[1]->ops.count, subs[0]->ops.sat + subs[1]->ops.sat, {}}; + case Fragment::AND_B: { + const auto count{1 + subs[0]->ops.count + subs[1]->ops.count}; + const auto sat{subs[0]->ops.sat + subs[1]->ops.sat}; + const auto dsat{subs[0]->ops.dsat + subs[1]->ops.dsat}; + return {count, sat, dsat}; + } + case Fragment::OR_B: { + const auto count{1 + subs[0]->ops.count + subs[1]->ops.count}; + const auto sat{(subs[0]->ops.sat + subs[1]->ops.dsat) | (subs[1]->ops.sat + subs[0]->ops.dsat)}; + const auto dsat{subs[0]->ops.dsat + subs[1]->ops.dsat}; + return {count, sat, dsat}; + } + case Fragment::OR_D: { + const auto count{3 + subs[0]->ops.count + subs[1]->ops.count}; + const auto sat{subs[0]->ops.sat | (subs[1]->ops.sat + subs[0]->ops.dsat)}; + const auto dsat{subs[0]->ops.dsat + subs[1]->ops.dsat}; + return {count, sat, dsat}; + } + case Fragment::OR_C: { + const auto count{2 + subs[0]->ops.count + subs[1]->ops.count}; + const auto sat{subs[0]->ops.sat | (subs[1]->ops.sat + subs[0]->ops.dsat)}; + return {count, sat, {}}; + } + case Fragment::OR_I: { + const auto count{3 + subs[0]->ops.count + subs[1]->ops.count}; + const auto sat{subs[0]->ops.sat | subs[1]->ops.sat}; + const auto dsat{subs[0]->ops.dsat | subs[1]->ops.dsat}; + return {count, sat, dsat}; + } + case Fragment::ANDOR: { + const auto count{3 + subs[0]->ops.count + subs[1]->ops.count + subs[2]->ops.count}; + const auto sat{(subs[1]->ops.sat + subs[0]->ops.sat) | (subs[0]->ops.dsat + subs[2]->ops.sat)}; + const auto dsat{subs[0]->ops.dsat + subs[2]->ops.dsat}; + return {count, sat, dsat}; + } + case Fragment::MULTI: return {1, (uint32_t)keys.size(), (uint32_t)keys.size()}; + case Fragment::WRAP_S: + case Fragment::WRAP_C: + case Fragment::WRAP_N: return {1 + subs[0]->ops.count, subs[0]->ops.sat, subs[0]->ops.dsat}; + case Fragment::WRAP_A: return {2 + subs[0]->ops.count, subs[0]->ops.sat, subs[0]->ops.dsat}; + case Fragment::WRAP_D: return {3 + subs[0]->ops.count, subs[0]->ops.sat, 0}; + case Fragment::WRAP_J: return {4 + subs[0]->ops.count, subs[0]->ops.sat, 0}; + case Fragment::WRAP_V: return {subs[0]->ops.count + (subs[0]->GetType() << "x"_mst), subs[0]->ops.sat, {}}; + case Fragment::THRESH: { + uint32_t count = 0; + auto sats = Vector(internal::MaxInt<uint32_t>(0)); + for (const auto& sub : subs) { + count += sub->ops.count + 1; + auto next_sats = Vector(sats[0] + sub->ops.dsat); + for (size_t j = 1; j < sats.size(); ++j) next_sats.push_back((sats[j] + sub->ops.dsat) | (sats[j - 1] + sub->ops.sat)); + next_sats.push_back(sats[sats.size() - 1] + sub->ops.sat); + sats = std::move(next_sats); + } + assert(k <= sats.size()); + return {count, sats[k], sats[0]}; + } + } + assert(false); + return {0, {}, {}}; + } + + internal::StackSize CalcStackSize() const { + switch (nodetype) { + case Fragment::JUST_0: return {{}, 0}; + case Fragment::JUST_1: + case Fragment::OLDER: + case Fragment::AFTER: return {0, {}}; + case Fragment::PK_K: return {1, 1}; + case Fragment::PK_H: return {2, 2}; + case Fragment::SHA256: + case Fragment::RIPEMD160: + case Fragment::HASH256: + case Fragment::HASH160: return {1, {}}; + case Fragment::ANDOR: { + const auto sat{(subs[0]->ss.sat + subs[1]->ss.sat) | (subs[0]->ss.dsat + subs[2]->ss.sat)}; + const auto dsat{subs[0]->ss.dsat + subs[2]->ss.dsat}; + return {sat, dsat}; + } + case Fragment::AND_V: return {subs[0]->ss.sat + subs[1]->ss.sat, {}}; + case Fragment::AND_B: return {subs[0]->ss.sat + subs[1]->ss.sat, subs[0]->ss.dsat + subs[1]->ss.dsat}; + case Fragment::OR_B: { + const auto sat{(subs[0]->ss.dsat + subs[1]->ss.sat) | (subs[0]->ss.sat + subs[1]->ss.dsat)}; + const auto dsat{subs[0]->ss.dsat + subs[1]->ss.dsat}; + return {sat, dsat}; + } + case Fragment::OR_C: return {subs[0]->ss.sat | (subs[0]->ss.dsat + subs[1]->ss.sat), {}}; + case Fragment::OR_D: return {subs[0]->ss.sat | (subs[0]->ss.dsat + subs[1]->ss.sat), subs[0]->ss.dsat + subs[1]->ss.dsat}; + case Fragment::OR_I: return {(subs[0]->ss.sat + 1) | (subs[1]->ss.sat + 1), (subs[0]->ss.dsat + 1) | (subs[1]->ss.dsat + 1)}; + case Fragment::MULTI: return {k + 1, k + 1}; + case Fragment::WRAP_A: + case Fragment::WRAP_N: + case Fragment::WRAP_S: + case Fragment::WRAP_C: return subs[0]->ss; + case Fragment::WRAP_D: return {1 + subs[0]->ss.sat, 1}; + case Fragment::WRAP_V: return {subs[0]->ss.sat, {}}; + case Fragment::WRAP_J: return {subs[0]->ss.sat, 1}; + case Fragment::THRESH: { + auto sats = Vector(internal::MaxInt<uint32_t>(0)); + for (const auto& sub : subs) { + auto next_sats = Vector(sats[0] + sub->ss.dsat); + for (size_t j = 1; j < sats.size(); ++j) next_sats.push_back((sats[j] + sub->ss.dsat) | (sats[j - 1] + sub->ss.sat)); + next_sats.push_back(sats[sats.size() - 1] + sub->ss.sat); + sats = std::move(next_sats); + } + assert(k <= sats.size()); + return {sats[k], sats[0]}; + } + } + assert(false); + return {{}, {}}; + } + +public: + //! Return the size of the script for this expression (faster than ToScript().size()). + size_t ScriptSize() const { return scriptlen; } + + //! Return the maximum number of ops needed to satisfy this script non-malleably. + uint32_t GetOps() const { return ops.count + ops.sat.value; } + + //! Check the ops limit of this script against the consensus limit. + bool CheckOpsLimit() const { return GetOps() <= MAX_OPS_PER_SCRIPT; } + + /** Return the maximum number of stack elements needed to satisfy this script non-malleably, including + * the script push. */ + uint32_t GetStackSize() const { return ss.sat.value + 1; } + + //! Check the maximum stack size for this script against the policy limit. + bool CheckStackSize() const { return GetStackSize() - 1 <= MAX_STANDARD_P2WSH_STACK_ITEMS; } + + //! Return the expression type. + Type GetType() const { return typ; } + + //! Check whether this node is valid at all. + bool IsValid() const { return !(GetType() == ""_mst) && ScriptSize() <= MAX_STANDARD_P2WSH_SCRIPT_SIZE; } + + //! Check whether this node is valid as a script on its own. + bool IsValidTopLevel() const { return IsValid() && GetType() << "B"_mst; } + + //! Check whether this script can always be satisfied in a non-malleable way. + bool IsNonMalleable() const { return GetType() << "m"_mst; } + + //! Check whether this script always needs a signature. + bool NeedsSignature() const { return GetType() << "s"_mst; } + + //! Do all sanity checks. + bool IsSane() const { return IsValid() && GetType() << "mk"_mst && CheckOpsLimit() && CheckStackSize(); } + + //! Check whether this node is safe as a script on its own. + bool IsSaneTopLevel() const { return IsValidTopLevel() && IsSane() && NeedsSignature(); } + + //! Equality testing. + bool operator==(const Node<Key>& arg) const + { + if (nodetype != arg.nodetype) return false; + if (k != arg.k) return false; + if (data != arg.data) return false; + if (keys != arg.keys) return false; + if (subs.size() != arg.subs.size()) return false; + for (size_t i = 0; i < subs.size(); ++i) { + if (!(*subs[i] == *arg.subs[i])) return false; + } + assert(scriptlen == arg.scriptlen); + assert(typ == arg.typ); + return true; + } + + // Constructors with various argument combinations. + Node(Fragment nt, std::vector<NodeRef<Key>> sub, std::vector<unsigned char> arg, uint32_t val = 0) : nodetype(nt), k(val), data(std::move(arg)), subs(std::move(sub)), ops(CalcOps()), ss(CalcStackSize()), typ(CalcType()), scriptlen(CalcScriptLen()) {} + Node(Fragment nt, std::vector<unsigned char> arg, uint32_t val = 0) : nodetype(nt), k(val), data(std::move(arg)), ops(CalcOps()), ss(CalcStackSize()), typ(CalcType()), scriptlen(CalcScriptLen()) {} + Node(Fragment nt, std::vector<NodeRef<Key>> sub, std::vector<Key> key, uint32_t val = 0) : nodetype(nt), k(val), keys(std::move(key)), subs(std::move(sub)), ops(CalcOps()), ss(CalcStackSize()), typ(CalcType()), scriptlen(CalcScriptLen()) {} + Node(Fragment nt, std::vector<Key> key, uint32_t val = 0) : nodetype(nt), k(val), keys(std::move(key)), ops(CalcOps()), ss(CalcStackSize()), typ(CalcType()), scriptlen(CalcScriptLen()) {} + Node(Fragment nt, std::vector<NodeRef<Key>> sub, uint32_t val = 0) : nodetype(nt), k(val), subs(std::move(sub)), ops(CalcOps()), ss(CalcStackSize()), typ(CalcType()), scriptlen(CalcScriptLen()) {} + Node(Fragment nt, uint32_t val = 0) : nodetype(nt), k(val), ops(CalcOps()), ss(CalcStackSize()), typ(CalcType()), scriptlen(CalcScriptLen()) {} +}; + +namespace internal { + +enum class ParseContext { + /** An expression which may be begin with wrappers followed by a colon. */ + WRAPPED_EXPR, + /** A miniscript expression which does not begin with wrappers. */ + EXPR, + + /** SWAP wraps the top constructed node with s: */ + SWAP, + /** ALT wraps the top constructed node with a: */ + ALT, + /** CHECK wraps the top constructed node with c: */ + CHECK, + /** DUP_IF wraps the top constructed node with d: */ + DUP_IF, + /** VERIFY wraps the top constructed node with v: */ + VERIFY, + /** NON_ZERO wraps the top constructed node with j: */ + NON_ZERO, + /** ZERO_NOTEQUAL wraps the top constructed node with n: */ + ZERO_NOTEQUAL, + /** WRAP_U will construct an or_i(X,0) node from the top constructed node. */ + WRAP_U, + /** WRAP_T will construct an and_v(X,1) node from the top constructed node. */ + WRAP_T, + + /** AND_N will construct an andor(X,Y,0) node from the last two constructed nodes. */ + AND_N, + /** AND_V will construct an and_v node from the last two constructed nodes. */ + AND_V, + /** AND_B will construct an and_b node from the last two constructed nodes. */ + AND_B, + /** ANDOR will construct an andor node from the last three constructed nodes. */ + ANDOR, + /** OR_B will construct an or_b node from the last two constructed nodes. */ + OR_B, + /** OR_C will construct an or_c node from the last two constructed nodes. */ + OR_C, + /** OR_D will construct an or_d node from the last two constructed nodes. */ + OR_D, + /** OR_I will construct an or_i node from the last two constructed nodes. */ + OR_I, + + /** THRESH will read a wrapped expression, and then look for a COMMA. If + * no comma follows, it will construct a thresh node from the appropriate + * number of constructed children. Otherwise, it will recurse with another + * THRESH. */ + THRESH, + + /** COMMA expects the next element to be ',' and fails if not. */ + COMMA, + /** CLOSE_BRACKET expects the next element to be ')' and fails if not. */ + CLOSE_BRACKET, +}; + +int FindNextChar(Span<const char> in, const char m); + +/** Parse a key string ending with a ')' or ','. */ +template<typename Key, typename Ctx> +std::optional<std::pair<Key, int>> ParseKeyEnd(Span<const char> in, const Ctx& ctx) +{ + Key key; + int key_size = FindNextChar(in, ')'); + if (key_size < 1) return {}; + if (!ctx.FromString(in.begin(), in.begin() + key_size, key)) return {}; + return {{std::move(key), key_size}}; +} + +/** Parse a hex string ending at the end of the fragment's text representation. */ +template<typename Ctx> +std::optional<std::pair<std::vector<unsigned char>, int>> ParseHexStrEnd(Span<const char> in, const size_t expected_size, + const Ctx& ctx) +{ + int hash_size = FindNextChar(in, ')'); + if (hash_size < 1) return {}; + std::string val = std::string(in.begin(), in.begin() + hash_size); + if (!IsHex(val)) return {}; + auto hash = ParseHex(val); + if (hash.size() != expected_size) return {}; + return {{std::move(hash), hash_size}}; +} + +/** BuildBack pops the last two elements off `constructed` and wraps them in the specified Fragment */ +template<typename Key> +void BuildBack(Fragment nt, std::vector<NodeRef<Key>>& constructed, const bool reverse = false) +{ + NodeRef<Key> child = std::move(constructed.back()); + constructed.pop_back(); + if (reverse) { + constructed.back() = MakeNodeRef<Key>(nt, Vector(std::move(child), std::move(constructed.back()))); + } else { + constructed.back() = MakeNodeRef<Key>(nt, Vector(std::move(constructed.back()), std::move(child))); + } +} + +//! Parse a miniscript from its textual descriptor form. +template<typename Key, typename Ctx> +inline NodeRef<Key> Parse(Span<const char> in, const Ctx& ctx) +{ + using namespace spanparsing; + + // The two integers are used to hold state for thresh() + std::vector<std::tuple<ParseContext, int64_t, int64_t>> to_parse; + std::vector<NodeRef<Key>> constructed; + + to_parse.emplace_back(ParseContext::WRAPPED_EXPR, -1, -1); + + while (!to_parse.empty()) { + // Get the current context we are decoding within + auto [cur_context, n, k] = to_parse.back(); + to_parse.pop_back(); + + switch (cur_context) { + case ParseContext::WRAPPED_EXPR: { + int colon_index = -1; + for (int i = 1; i < (int)in.size(); ++i) { + if (in[i] == ':') { + colon_index = i; + break; + } + if (in[i] < 'a' || in[i] > 'z') break; + } + // If there is no colon, this loop won't execute + for (int j = 0; j < colon_index; ++j) { + if (in[j] == 'a') { + to_parse.emplace_back(ParseContext::ALT, -1, -1); + } else if (in[j] == 's') { + to_parse.emplace_back(ParseContext::SWAP, -1, -1); + } else if (in[j] == 'c') { + to_parse.emplace_back(ParseContext::CHECK, -1, -1); + } else if (in[j] == 'd') { + to_parse.emplace_back(ParseContext::DUP_IF, -1, -1); + } else if (in[j] == 'j') { + to_parse.emplace_back(ParseContext::NON_ZERO, -1, -1); + } else if (in[j] == 'n') { + to_parse.emplace_back(ParseContext::ZERO_NOTEQUAL, -1, -1); + } else if (in[j] == 'v') { + to_parse.emplace_back(ParseContext::VERIFY, -1, -1); + } else if (in[j] == 'u') { + to_parse.emplace_back(ParseContext::WRAP_U, -1, -1); + } else if (in[j] == 't') { + to_parse.emplace_back(ParseContext::WRAP_T, -1, -1); + } else if (in[j] == 'l') { + // The l: wrapper is equivalent to or_i(0,X) + constructed.push_back(MakeNodeRef<Key>(Fragment::JUST_0)); + to_parse.emplace_back(ParseContext::OR_I, -1, -1); + } else { + return {}; + } + } + to_parse.emplace_back(ParseContext::EXPR, -1, -1); + in = in.subspan(colon_index + 1); + break; + } + case ParseContext::EXPR: { + if (Const("0", in)) { + constructed.push_back(MakeNodeRef<Key>(Fragment::JUST_0)); + } else if (Const("1", in)) { + constructed.push_back(MakeNodeRef<Key>(Fragment::JUST_1)); + } else if (Const("pk(", in)) { + auto res = ParseKeyEnd<Key, Ctx>(in, ctx); + if (!res) return {}; + auto& [key, key_size] = *res; + constructed.push_back(MakeNodeRef<Key>(Fragment::WRAP_C, Vector(MakeNodeRef<Key>(Fragment::PK_K, Vector(std::move(key)))))); + in = in.subspan(key_size + 1); + } else if (Const("pkh(", in)) { + auto res = ParseKeyEnd<Key>(in, ctx); + if (!res) return {}; + auto& [key, key_size] = *res; + constructed.push_back(MakeNodeRef<Key>(Fragment::WRAP_C, Vector(MakeNodeRef<Key>(Fragment::PK_H, Vector(std::move(key)))))); + in = in.subspan(key_size + 1); + } else if (Const("pk_k(", in)) { + auto res = ParseKeyEnd<Key>(in, ctx); + if (!res) return {}; + auto& [key, key_size] = *res; + constructed.push_back(MakeNodeRef<Key>(Fragment::PK_K, Vector(std::move(key)))); + in = in.subspan(key_size + 1); + } else if (Const("pk_h(", in)) { + auto res = ParseKeyEnd<Key>(in, ctx); + if (!res) return {}; + auto& [key, key_size] = *res; + constructed.push_back(MakeNodeRef<Key>(Fragment::PK_H, Vector(std::move(key)))); + in = in.subspan(key_size + 1); + } else if (Const("sha256(", in)) { + auto res = ParseHexStrEnd(in, 32, ctx); + if (!res) return {}; + auto& [hash, hash_size] = *res; + constructed.push_back(MakeNodeRef<Key>(Fragment::SHA256, std::move(hash))); + in = in.subspan(hash_size + 1); + } else if (Const("ripemd160(", in)) { + auto res = ParseHexStrEnd(in, 20, ctx); + if (!res) return {}; + auto& [hash, hash_size] = *res; + constructed.push_back(MakeNodeRef<Key>(Fragment::RIPEMD160, std::move(hash))); + in = in.subspan(hash_size + 1); + } else if (Const("hash256(", in)) { + auto res = ParseHexStrEnd(in, 32, ctx); + if (!res) return {}; + auto& [hash, hash_size] = *res; + constructed.push_back(MakeNodeRef<Key>(Fragment::HASH256, std::move(hash))); + in = in.subspan(hash_size + 1); + } else if (Const("hash160(", in)) { + auto res = ParseHexStrEnd(in, 20, ctx); + if (!res) return {}; + auto& [hash, hash_size] = *res; + constructed.push_back(MakeNodeRef<Key>(Fragment::HASH160, std::move(hash))); + in = in.subspan(hash_size + 1); + } else if (Const("after(", in)) { + int arg_size = FindNextChar(in, ')'); + if (arg_size < 1) return {}; + int64_t num; + if (!ParseInt64(std::string(in.begin(), in.begin() + arg_size), &num)) return {}; + if (num < 1 || num >= 0x80000000L) return {}; + constructed.push_back(MakeNodeRef<Key>(Fragment::AFTER, num)); + in = in.subspan(arg_size + 1); + } else if (Const("older(", in)) { + int arg_size = FindNextChar(in, ')'); + if (arg_size < 1) return {}; + int64_t num; + if (!ParseInt64(std::string(in.begin(), in.begin() + arg_size), &num)) return {}; + if (num < 1 || num >= 0x80000000L) return {}; + constructed.push_back(MakeNodeRef<Key>(Fragment::OLDER, num)); + in = in.subspan(arg_size + 1); + } else if (Const("multi(", in)) { + // Get threshold + int next_comma = FindNextChar(in, ','); + if (next_comma < 1) return {}; + if (!ParseInt64(std::string(in.begin(), in.begin() + next_comma), &k)) return {}; + in = in.subspan(next_comma + 1); + // Get keys + std::vector<Key> keys; + while (next_comma != -1) { + Key key; + next_comma = FindNextChar(in, ','); + int key_length = (next_comma == -1) ? FindNextChar(in, ')') : next_comma; + if (key_length < 1) return {}; + if (!ctx.FromString(in.begin(), in.begin() + key_length, key)) return {}; + keys.push_back(std::move(key)); + in = in.subspan(key_length + 1); + } + if (keys.size() < 1 || keys.size() > 20) return {}; + if (k < 1 || k > (int64_t)keys.size()) return {}; + constructed.push_back(MakeNodeRef<Key>(Fragment::MULTI, std::move(keys), k)); + } else if (Const("thresh(", in)) { + int next_comma = FindNextChar(in, ','); + if (next_comma < 1) return {}; + if (!ParseInt64(std::string(in.begin(), in.begin() + next_comma), &k)) return {}; + if (k < 1) return {}; + in = in.subspan(next_comma + 1); + // n = 1 here because we read the first WRAPPED_EXPR before reaching THRESH + to_parse.emplace_back(ParseContext::THRESH, 1, k); + to_parse.emplace_back(ParseContext::WRAPPED_EXPR, -1, -1); + } else if (Const("andor(", in)) { + to_parse.emplace_back(ParseContext::ANDOR, -1, -1); + to_parse.emplace_back(ParseContext::CLOSE_BRACKET, -1, -1); + to_parse.emplace_back(ParseContext::WRAPPED_EXPR, -1, -1); + to_parse.emplace_back(ParseContext::COMMA, -1, -1); + to_parse.emplace_back(ParseContext::WRAPPED_EXPR, -1, -1); + to_parse.emplace_back(ParseContext::COMMA, -1, -1); + to_parse.emplace_back(ParseContext::WRAPPED_EXPR, -1, -1); + } else { + if (Const("and_n(", in)) { + to_parse.emplace_back(ParseContext::AND_N, -1, -1); + } else if (Const("and_b(", in)) { + to_parse.emplace_back(ParseContext::AND_B, -1, -1); + } else if (Const("and_v(", in)) { + to_parse.emplace_back(ParseContext::AND_V, -1, -1); + } else if (Const("or_b(", in)) { + to_parse.emplace_back(ParseContext::OR_B, -1, -1); + } else if (Const("or_c(", in)) { + to_parse.emplace_back(ParseContext::OR_C, -1, -1); + } else if (Const("or_d(", in)) { + to_parse.emplace_back(ParseContext::OR_D, -1, -1); + } else if (Const("or_i(", in)) { + to_parse.emplace_back(ParseContext::OR_I, -1, -1); + } else { + return {}; + } + to_parse.emplace_back(ParseContext::CLOSE_BRACKET, -1, -1); + to_parse.emplace_back(ParseContext::WRAPPED_EXPR, -1, -1); + to_parse.emplace_back(ParseContext::COMMA, -1, -1); + to_parse.emplace_back(ParseContext::WRAPPED_EXPR, -1, -1); + } + break; + } + case ParseContext::ALT: { + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_A, Vector(std::move(constructed.back()))); + break; + } + case ParseContext::SWAP: { + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_S, Vector(std::move(constructed.back()))); + break; + } + case ParseContext::CHECK: { + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_C, Vector(std::move(constructed.back()))); + break; + } + case ParseContext::DUP_IF: { + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_D, Vector(std::move(constructed.back()))); + break; + } + case ParseContext::NON_ZERO: { + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_J, Vector(std::move(constructed.back()))); + break; + } + case ParseContext::ZERO_NOTEQUAL: { + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_N, Vector(std::move(constructed.back()))); + break; + } + case ParseContext::VERIFY: { + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_V, Vector(std::move(constructed.back()))); + break; + } + case ParseContext::WRAP_U: { + constructed.back() = MakeNodeRef<Key>(Fragment::OR_I, Vector(std::move(constructed.back()), MakeNodeRef<Key>(Fragment::JUST_0))); + break; + } + case ParseContext::WRAP_T: { + constructed.back() = MakeNodeRef<Key>(Fragment::AND_V, Vector(std::move(constructed.back()), MakeNodeRef<Key>(Fragment::JUST_1))); + break; + } + case ParseContext::AND_B: { + BuildBack(Fragment::AND_B, constructed); + break; + } + case ParseContext::AND_N: { + auto mid = std::move(constructed.back()); + constructed.pop_back(); + constructed.back() = MakeNodeRef<Key>(Fragment::ANDOR, Vector(std::move(constructed.back()), std::move(mid), MakeNodeRef<Key>(Fragment::JUST_0))); + break; + } + case ParseContext::AND_V: { + BuildBack(Fragment::AND_V, constructed); + break; + } + case ParseContext::OR_B: { + BuildBack(Fragment::OR_B, constructed); + break; + } + case ParseContext::OR_C: { + BuildBack(Fragment::OR_C, constructed); + break; + } + case ParseContext::OR_D: { + BuildBack(Fragment::OR_D, constructed); + break; + } + case ParseContext::OR_I: { + BuildBack(Fragment::OR_I, constructed); + break; + } + case ParseContext::ANDOR: { + auto right = std::move(constructed.back()); + constructed.pop_back(); + auto mid = std::move(constructed.back()); + constructed.pop_back(); + constructed.back() = MakeNodeRef<Key>(Fragment::ANDOR, Vector(std::move(constructed.back()), std::move(mid), std::move(right))); + break; + } + case ParseContext::THRESH: { + if (in.size() < 1) return {}; + if (in[0] == ',') { + in = in.subspan(1); + to_parse.emplace_back(ParseContext::THRESH, n+1, k); + to_parse.emplace_back(ParseContext::WRAPPED_EXPR, -1, -1); + } else if (in[0] == ')') { + if (k > n) return {}; + in = in.subspan(1); + // Children are constructed in reverse order, so iterate from end to beginning + std::vector<NodeRef<Key>> subs; + for (int i = 0; i < n; ++i) { + subs.push_back(std::move(constructed.back())); + constructed.pop_back(); + } + std::reverse(subs.begin(), subs.end()); + constructed.push_back(MakeNodeRef<Key>(Fragment::THRESH, std::move(subs), k)); + } else { + return {}; + } + break; + } + case ParseContext::COMMA: { + if (in.size() < 1 || in[0] != ',') return {}; + in = in.subspan(1); + break; + } + case ParseContext::CLOSE_BRACKET: { + if (in.size() < 1 || in[0] != ')') return {}; + in = in.subspan(1); + break; + } + } + } + + // Sanity checks on the produced miniscript + assert(constructed.size() == 1); + if (in.size() > 0) return {}; + const NodeRef<Key> tl_node = std::move(constructed.front()); + if (!tl_node->IsValidTopLevel()) return {}; + return tl_node; +} + +/** Decode a script into opcode/push pairs. + * + * Construct a vector with one element per opcode in the script, in reverse order. + * Each element is a pair consisting of the opcode, as well as the data pushed by + * the opcode (including OP_n), if any. OP_CHECKSIGVERIFY, OP_CHECKMULTISIGVERIFY, + * and OP_EQUALVERIFY are decomposed into OP_CHECKSIG, OP_CHECKMULTISIG, OP_EQUAL + * respectively, plus OP_VERIFY. + */ +bool DecomposeScript(const CScript& script, std::vector<std::pair<opcodetype, std::vector<unsigned char>>>& out); + +/** Determine whether the passed pair (created by DecomposeScript) is pushing a number. */ +bool ParseScriptNumber(const std::pair<opcodetype, std::vector<unsigned char>>& in, int64_t& k); + +enum class DecodeContext { + /** A single expression of type B, K, or V. Specifically, this can't be an + * and_v or an expression of type W (a: and s: wrappers). */ + SINGLE_BKV_EXPR, + /** Potentially multiple SINGLE_BKV_EXPRs as children of (potentially multiple) + * and_v expressions. Syntactic sugar for MAYBE_AND_V + SINGLE_BKV_EXPR. */ + BKV_EXPR, + /** An expression of type W (a: or s: wrappers). */ + W_EXPR, + + /** SWAP expects the next element to be OP_SWAP (inside a W-type expression that + * didn't end with FROMALTSTACK), and wraps the top of the constructed stack + * with s: */ + SWAP, + /** ALT expects the next element to be TOALTSTACK (we must have already read a + * FROMALTSTACK earlier), and wraps the top of the constructed stack with a: */ + ALT, + /** CHECK wraps the top constructed node with c: */ + CHECK, + /** DUP_IF wraps the top constructed node with d: */ + DUP_IF, + /** VERIFY wraps the top constructed node with v: */ + VERIFY, + /** NON_ZERO wraps the top constructed node with j: */ + NON_ZERO, + /** ZERO_NOTEQUAL wraps the top constructed node with n: */ + ZERO_NOTEQUAL, + + /** MAYBE_AND_V will check if the next part of the script could be a valid + * miniscript sub-expression, and if so it will push AND_V and SINGLE_BKV_EXPR + * to decode it and construct the and_v node. This is recursive, to deal with + * multiple and_v nodes inside each other. */ + MAYBE_AND_V, + /** AND_V will construct an and_v node from the last two constructed nodes. */ + AND_V, + /** AND_B will construct an and_b node from the last two constructed nodes. */ + AND_B, + /** ANDOR will construct an andor node from the last three constructed nodes. */ + ANDOR, + /** OR_B will construct an or_b node from the last two constructed nodes. */ + OR_B, + /** OR_C will construct an or_c node from the last two constructed nodes. */ + OR_C, + /** OR_D will construct an or_d node from the last two constructed nodes. */ + OR_D, + + /** In a thresh expression, all sub-expressions other than the first are W-type, + * and end in OP_ADD. THRESH_W will check for this OP_ADD and either push a W_EXPR + * or a SINGLE_BKV_EXPR and jump to THRESH_E accordingly. */ + THRESH_W, + /** THRESH_E constructs a thresh node from the appropriate number of constructed + * children. */ + THRESH_E, + + /** ENDIF signals that we are inside some sort of OP_IF structure, which could be + * or_d, or_c, or_i, andor, d:, or j: wrapper, depending on what follows. We read + * a BKV_EXPR and then deal with the next opcode case-by-case. */ + ENDIF, + /** If, inside an ENDIF context, we find an OP_NOTIF before finding an OP_ELSE, + * we could either be in an or_d or an or_c node. We then check for IFDUP to + * distinguish these cases. */ + ENDIF_NOTIF, + /** If, inside an ENDIF context, we find an OP_ELSE, then we could be in either an + * or_i or an andor node. Read the next BKV_EXPR and find either an OP_IF or an + * OP_NOTIF. */ + ENDIF_ELSE, +}; + +//! Parse a miniscript from a bitcoin script +template<typename Key, typename Ctx, typename I> +inline NodeRef<Key> DecodeScript(I& in, I last, const Ctx& ctx) +{ + // The two integers are used to hold state for thresh() + std::vector<std::tuple<DecodeContext, int64_t, int64_t>> to_parse; + std::vector<NodeRef<Key>> constructed; + + // This is the top level, so we assume the type is B + // (in particular, disallowing top level W expressions) + to_parse.emplace_back(DecodeContext::BKV_EXPR, -1, -1); + + while (!to_parse.empty()) { + // Exit early if the Miniscript is not going to be valid. + if (!constructed.empty() && !constructed.back()->IsValid()) return {}; + + // Get the current context we are decoding within + auto [cur_context, n, k] = to_parse.back(); + to_parse.pop_back(); + + switch(cur_context) { + case DecodeContext::SINGLE_BKV_EXPR: { + if (in >= last) return {}; + + // Constants + if (in[0].first == OP_1) { + ++in; + constructed.push_back(MakeNodeRef<Key>(Fragment::JUST_1)); + break; + } + if (in[0].first == OP_0) { + ++in; + constructed.push_back(MakeNodeRef<Key>(Fragment::JUST_0)); + break; + } + // Public keys + if (in[0].second.size() == 33) { + Key key; + if (!ctx.FromPKBytes(in[0].second.begin(), in[0].second.end(), key)) return {}; + ++in; + constructed.push_back(MakeNodeRef<Key>(Fragment::PK_K, Vector(std::move(key)))); + break; + } + if (last - in >= 5 && in[0].first == OP_VERIFY && in[1].first == OP_EQUAL && in[3].first == OP_HASH160 && in[4].first == OP_DUP && in[2].second.size() == 20) { + Key key; + if (!ctx.FromPKHBytes(in[2].second.begin(), in[2].second.end(), key)) return {}; + in += 5; + constructed.push_back(MakeNodeRef<Key>(Fragment::PK_H, Vector(std::move(key)))); + break; + } + // Time locks + if (last - in >= 2 && in[0].first == OP_CHECKSEQUENCEVERIFY && ParseScriptNumber(in[1], k)) { + in += 2; + if (k < 1 || k > 0x7FFFFFFFL) return {}; + constructed.push_back(MakeNodeRef<Key>(Fragment::OLDER, k)); + break; + } + if (last - in >= 2 && in[0].first == OP_CHECKLOCKTIMEVERIFY && ParseScriptNumber(in[1], k)) { + in += 2; + if (k < 1 || k > 0x7FFFFFFFL) return {}; + constructed.push_back(MakeNodeRef<Key>(Fragment::AFTER, k)); + break; + } + // Hashes + if (last - in >= 7 && in[0].first == OP_EQUAL && in[3].first == OP_VERIFY && in[4].first == OP_EQUAL && ParseScriptNumber(in[5], k) && k == 32 && in[6].first == OP_SIZE) { + if (in[2].first == OP_SHA256 && in[1].second.size() == 32) { + constructed.push_back(MakeNodeRef<Key>(Fragment::SHA256, in[1].second)); + in += 7; + break; + } else if (in[2].first == OP_RIPEMD160 && in[1].second.size() == 20) { + constructed.push_back(MakeNodeRef<Key>(Fragment::RIPEMD160, in[1].second)); + in += 7; + break; + } else if (in[2].first == OP_HASH256 && in[1].second.size() == 32) { + constructed.push_back(MakeNodeRef<Key>(Fragment::HASH256, in[1].second)); + in += 7; + break; + } else if (in[2].first == OP_HASH160 && in[1].second.size() == 20) { + constructed.push_back(MakeNodeRef<Key>(Fragment::HASH160, in[1].second)); + in += 7; + break; + } + } + // Multi + if (last - in >= 3 && in[0].first == OP_CHECKMULTISIG) { + std::vector<Key> keys; + if (!ParseScriptNumber(in[1], n)) return {}; + if (last - in < 3 + n) return {}; + if (n < 1 || n > 20) return {}; + for (int i = 0; i < n; ++i) { + Key key; + if (in[2 + i].second.size() != 33) return {}; + if (!ctx.FromPKBytes(in[2 + i].second.begin(), in[2 + i].second.end(), key)) return {}; + keys.push_back(std::move(key)); + } + if (!ParseScriptNumber(in[2 + n], k)) return {}; + if (k < 1 || k > n) return {}; + in += 3 + n; + std::reverse(keys.begin(), keys.end()); + constructed.push_back(MakeNodeRef<Key>(Fragment::MULTI, std::move(keys), k)); + break; + } + /** In the following wrappers, we only need to push SINGLE_BKV_EXPR rather + * than BKV_EXPR, because and_v commutes with these wrappers. For example, + * c:and_v(X,Y) produces the same script as and_v(X,c:Y). */ + // c: wrapper + if (in[0].first == OP_CHECKSIG) { + ++in; + to_parse.emplace_back(DecodeContext::CHECK, -1, -1); + to_parse.emplace_back(DecodeContext::SINGLE_BKV_EXPR, -1, -1); + break; + } + // v: wrapper + if (in[0].first == OP_VERIFY) { + ++in; + to_parse.emplace_back(DecodeContext::VERIFY, -1, -1); + to_parse.emplace_back(DecodeContext::SINGLE_BKV_EXPR, -1, -1); + break; + } + // n: wrapper + if (in[0].first == OP_0NOTEQUAL) { + ++in; + to_parse.emplace_back(DecodeContext::ZERO_NOTEQUAL, -1, -1); + to_parse.emplace_back(DecodeContext::SINGLE_BKV_EXPR, -1, -1); + break; + } + // Thresh + if (last - in >= 3 && in[0].first == OP_EQUAL && ParseScriptNumber(in[1], k)) { + if (k < 1) return {}; + in += 2; + to_parse.emplace_back(DecodeContext::THRESH_W, 0, k); + break; + } + // OP_ENDIF can be WRAP_J, WRAP_D, ANDOR, OR_C, OR_D, or OR_I + if (in[0].first == OP_ENDIF) { + ++in; + to_parse.emplace_back(DecodeContext::ENDIF, -1, -1); + to_parse.emplace_back(DecodeContext::BKV_EXPR, -1, -1); + break; + } + /** In and_b and or_b nodes, we only look for SINGLE_BKV_EXPR, because + * or_b(and_v(X,Y),Z) has script [X] [Y] [Z] OP_BOOLOR, the same as + * and_v(X,or_b(Y,Z)). In this example, the former of these is invalid as + * miniscript, while the latter is valid. So we leave the and_v "outside" + * while decoding. */ + // and_b + if (in[0].first == OP_BOOLAND) { + ++in; + to_parse.emplace_back(DecodeContext::AND_B, -1, -1); + to_parse.emplace_back(DecodeContext::SINGLE_BKV_EXPR, -1, -1); + to_parse.emplace_back(DecodeContext::W_EXPR, -1, -1); + break; + } + // or_b + if (in[0].first == OP_BOOLOR) { + ++in; + to_parse.emplace_back(DecodeContext::OR_B, -1, -1); + to_parse.emplace_back(DecodeContext::SINGLE_BKV_EXPR, -1, -1); + to_parse.emplace_back(DecodeContext::W_EXPR, -1, -1); + break; + } + // Unrecognised expression + return {}; + } + case DecodeContext::BKV_EXPR: { + to_parse.emplace_back(DecodeContext::MAYBE_AND_V, -1, -1); + to_parse.emplace_back(DecodeContext::SINGLE_BKV_EXPR, -1, -1); + break; + } + case DecodeContext::W_EXPR: { + // a: wrapper + if (in >= last) return {}; + if (in[0].first == OP_FROMALTSTACK) { + ++in; + to_parse.emplace_back(DecodeContext::ALT, -1, -1); + } else { + to_parse.emplace_back(DecodeContext::SWAP, -1, -1); + } + to_parse.emplace_back(DecodeContext::BKV_EXPR, -1, -1); + break; + } + case DecodeContext::MAYBE_AND_V: { + // If we reach a potential AND_V top-level, check if the next part of the script could be another AND_V child + // These op-codes cannot end any well-formed miniscript so cannot be used in an and_v node. + if (in < last && in[0].first != OP_IF && in[0].first != OP_ELSE && in[0].first != OP_NOTIF && in[0].first != OP_TOALTSTACK && in[0].first != OP_SWAP) { + to_parse.emplace_back(DecodeContext::AND_V, -1, -1); + // BKV_EXPR can contain more AND_V nodes + to_parse.emplace_back(DecodeContext::BKV_EXPR, -1, -1); + } + break; + } + case DecodeContext::SWAP: { + if (in >= last || in[0].first != OP_SWAP || constructed.empty()) return {}; + ++in; + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_S, Vector(std::move(constructed.back()))); + break; + } + case DecodeContext::ALT: { + if (in >= last || in[0].first != OP_TOALTSTACK || constructed.empty()) return {}; + ++in; + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_A, Vector(std::move(constructed.back()))); + break; + } + case DecodeContext::CHECK: { + if (constructed.empty()) return {}; + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_C, Vector(std::move(constructed.back()))); + break; + } + case DecodeContext::DUP_IF: { + if (constructed.empty()) return {}; + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_D, Vector(std::move(constructed.back()))); + break; + } + case DecodeContext::VERIFY: { + if (constructed.empty()) return {}; + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_V, Vector(std::move(constructed.back()))); + break; + } + case DecodeContext::NON_ZERO: { + if (constructed.empty()) return {}; + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_J, Vector(std::move(constructed.back()))); + break; + } + case DecodeContext::ZERO_NOTEQUAL: { + if (constructed.empty()) return {}; + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_N, Vector(std::move(constructed.back()))); + break; + } + case DecodeContext::AND_V: { + if (constructed.size() < 2) return {}; + BuildBack(Fragment::AND_V, constructed, /*reverse=*/true); + break; + } + case DecodeContext::AND_B: { + if (constructed.size() < 2) return {}; + BuildBack(Fragment::AND_B, constructed, /*reverse=*/true); + break; + } + case DecodeContext::OR_B: { + if (constructed.size() < 2) return {}; + BuildBack(Fragment::OR_B, constructed, /*reverse=*/true); + break; + } + case DecodeContext::OR_C: { + if (constructed.size() < 2) return {}; + BuildBack(Fragment::OR_C, constructed, /*reverse=*/true); + break; + } + case DecodeContext::OR_D: { + if (constructed.size() < 2) return {}; + BuildBack(Fragment::OR_D, constructed, /*reverse=*/true); + break; + } + case DecodeContext::ANDOR: { + if (constructed.size() < 3) return {}; + NodeRef<Key> left = std::move(constructed.back()); + constructed.pop_back(); + NodeRef<Key> right = std::move(constructed.back()); + constructed.pop_back(); + NodeRef<Key> mid = std::move(constructed.back()); + constructed.back() = MakeNodeRef<Key>(Fragment::ANDOR, Vector(std::move(left), std::move(mid), std::move(right))); + break; + } + case DecodeContext::THRESH_W: { + if (in >= last) return {}; + if (in[0].first == OP_ADD) { + ++in; + to_parse.emplace_back(DecodeContext::THRESH_W, n+1, k); + to_parse.emplace_back(DecodeContext::W_EXPR, -1, -1); + } else { + to_parse.emplace_back(DecodeContext::THRESH_E, n+1, k); + // All children of thresh have type modifier d, so cannot be and_v + to_parse.emplace_back(DecodeContext::SINGLE_BKV_EXPR, -1, -1); + } + break; + } + case DecodeContext::THRESH_E: { + if (k < 1 || k > n || constructed.size() < static_cast<size_t>(n)) return {}; + std::vector<NodeRef<Key>> subs; + for (int i = 0; i < n; ++i) { + NodeRef<Key> sub = std::move(constructed.back()); + constructed.pop_back(); + subs.push_back(std::move(sub)); + } + constructed.push_back(MakeNodeRef<Key>(Fragment::THRESH, std::move(subs), k)); + break; + } + case DecodeContext::ENDIF: { + if (in >= last) return {}; + + // could be andor or or_i + if (in[0].first == OP_ELSE) { + ++in; + to_parse.emplace_back(DecodeContext::ENDIF_ELSE, -1, -1); + to_parse.emplace_back(DecodeContext::BKV_EXPR, -1, -1); + } + // could be j: or d: wrapper + else if (in[0].first == OP_IF) { + if (last - in >= 2 && in[1].first == OP_DUP) { + in += 2; + to_parse.emplace_back(DecodeContext::DUP_IF, -1, -1); + } else if (last - in >= 3 && in[1].first == OP_0NOTEQUAL && in[2].first == OP_SIZE) { + in += 3; + to_parse.emplace_back(DecodeContext::NON_ZERO, -1, -1); + } + else { + return {}; + } + // could be or_c or or_d + } else if (in[0].first == OP_NOTIF) { + ++in; + to_parse.emplace_back(DecodeContext::ENDIF_NOTIF, -1, -1); + } + else { + return {}; + } + break; + } + case DecodeContext::ENDIF_NOTIF: { + if (in >= last) return {}; + if (in[0].first == OP_IFDUP) { + ++in; + to_parse.emplace_back(DecodeContext::OR_D, -1, -1); + } else { + to_parse.emplace_back(DecodeContext::OR_C, -1, -1); + } + // or_c and or_d both require X to have type modifier d so, can't contain and_v + to_parse.emplace_back(DecodeContext::SINGLE_BKV_EXPR, -1, -1); + break; + } + case DecodeContext::ENDIF_ELSE: { + if (in >= last) return {}; + if (in[0].first == OP_IF) { + ++in; + BuildBack(Fragment::OR_I, constructed, /*reverse=*/true); + } else if (in[0].first == OP_NOTIF) { + ++in; + to_parse.emplace_back(DecodeContext::ANDOR, -1, -1); + // andor requires X to have type modifier d, so it can't be and_v + to_parse.emplace_back(DecodeContext::SINGLE_BKV_EXPR, -1, -1); + } else { + return {}; + } + break; + } + } + } + if (constructed.size() != 1) return {}; + const NodeRef<Key> tl_node = std::move(constructed.front()); + // Note that due to how ComputeType works (only assign the type to the node if the + // subs' types are valid) this would fail if any node of tree is badly typed. + if (!tl_node->IsValidTopLevel()) return {}; + return tl_node; +} + +} // namespace internal + +template<typename Ctx> +inline NodeRef<typename Ctx::Key> FromString(const std::string& str, const Ctx& ctx) { + return internal::Parse<typename Ctx::Key>(str, ctx); +} + +template<typename Ctx> +inline NodeRef<typename Ctx::Key> FromScript(const CScript& script, const Ctx& ctx) { + using namespace internal; + std::vector<std::pair<opcodetype, std::vector<unsigned char>>> decomposed; + if (!DecomposeScript(script, decomposed)) return {}; + auto it = decomposed.begin(); + auto ret = DecodeScript<typename Ctx::Key>(it, decomposed.end(), ctx); + if (!ret) return {}; + if (it != decomposed.end()) return {}; + return ret; +} + +} // namespace miniscript + +#endif // BITCOIN_SCRIPT_MINISCRIPT_H diff --git a/src/script/script.cpp b/src/script/script.cpp index 9a6419088b..88b4bc2f44 100644 --- a/src/script/script.cpp +++ b/src/script/script.cpp @@ -339,3 +339,28 @@ bool IsOpSuccess(const opcodetype& opcode) (opcode >= 141 && opcode <= 142) || (opcode >= 149 && opcode <= 153) || (opcode >= 187 && opcode <= 254); } + +bool CheckMinimalPush(const std::vector<unsigned char>& data, opcodetype opcode) { + // Excludes OP_1NEGATE, OP_1-16 since they are by definition minimal + assert(0 <= opcode && opcode <= OP_PUSHDATA4); + if (data.size() == 0) { + // Should have used OP_0. + return opcode == OP_0; + } else if (data.size() == 1 && data[0] >= 1 && data[0] <= 16) { + // Should have used OP_1 .. OP_16. + return false; + } else if (data.size() == 1 && data[0] == 0x81) { + // Should have used OP_1NEGATE. + return false; + } else if (data.size() <= 75) { + // Must have used a direct push (opcode indicating number of bytes pushed + those bytes). + return opcode == data.size(); + } else if (data.size() <= 255) { + // Must have used OP_PUSHDATA. + return opcode == OP_PUSHDATA1; + } else if (data.size() <= 65535) { + // Must have used OP_PUSHDATA2. + return opcode == OP_PUSHDATA2; + } + return true; +} diff --git a/src/script/script.h b/src/script/script.h index 8b7a7bb7b3..3b799ad637 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -29,6 +29,9 @@ static const int MAX_OPS_PER_SCRIPT = 201; // Maximum number of public keys per multisig static const int MAX_PUBKEYS_PER_MULTISIG = 20; +/** The limit of keys in OP_CHECKSIGADD-based scripts. It is due to the stack limit in BIP342. */ +static constexpr unsigned int MAX_PUBKEYS_PER_MULTI_A = 999; + // Maximum script length in bytes static const int MAX_SCRIPT_SIZE = 10000; @@ -332,6 +335,8 @@ public: return m_value; } + int64_t GetInt64() const { return m_value; } + std::vector<unsigned char> getvch() const { return serialize(m_value); @@ -573,4 +578,31 @@ struct CScriptWitness /** Test for OP_SUCCESSx opcodes as defined by BIP342. */ bool IsOpSuccess(const opcodetype& opcode); +bool CheckMinimalPush(const std::vector<unsigned char>& data, opcodetype opcode); + +/** Build a script by concatenating other scripts, or any argument accepted by CScript::operator<<. */ +template<typename... Ts> +CScript BuildScript(Ts&&... inputs) +{ + CScript ret; + int cnt{0}; + + ([&ret, &cnt] (Ts&& input) { + cnt++; + if constexpr (std::is_same_v<std::remove_cv_t<std::remove_reference_t<Ts>>, CScript>) { + // If it is a CScript, extend ret with it. Move or copy the first element instead. + if (cnt == 0) { + ret = std::forward<Ts>(input); + } else { + ret.insert(ret.end(), input.begin(), input.end()); + } + } else { + // Otherwise invoke CScript::operator<<. + ret << input; + } + } (std::forward<Ts>(inputs)), ...); + + return ret; +} + #endif // BITCOIN_SCRIPT_SCRIPT_H diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 371a937bc8..7328e8d1ad 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -18,16 +18,16 @@ typedef std::vector<unsigned char> valtype; -MutableTransactionSignatureCreator::MutableTransactionSignatureCreator(const CMutableTransaction* tx, unsigned int input_idx, const CAmount& amount, int hash_type) - : txTo{tx}, nIn{input_idx}, nHashType{hash_type}, amount{amount}, checker{txTo, nIn, amount, MissingDataBehavior::FAIL}, +MutableTransactionSignatureCreator::MutableTransactionSignatureCreator(const CMutableTransaction& tx, unsigned int input_idx, const CAmount& amount, int hash_type) + : m_txto{tx}, nIn{input_idx}, nHashType{hash_type}, amount{amount}, checker{&m_txto, nIn, amount, MissingDataBehavior::FAIL}, m_txdata(nullptr) { } -MutableTransactionSignatureCreator::MutableTransactionSignatureCreator(const CMutableTransaction* tx, unsigned int input_idx, const CAmount& amount, const PrecomputedTransactionData* txdata, int hash_type) - : txTo{tx}, nIn{input_idx}, nHashType{hash_type}, amount{amount}, - checker{txdata ? MutableTransactionSignatureChecker{txTo, nIn, amount, *txdata, MissingDataBehavior::FAIL} : - MutableTransactionSignatureChecker{txTo, nIn, amount, MissingDataBehavior::FAIL}}, +MutableTransactionSignatureCreator::MutableTransactionSignatureCreator(const CMutableTransaction& tx, unsigned int input_idx, const CAmount& amount, const PrecomputedTransactionData* txdata, int hash_type) + : m_txto{tx}, nIn{input_idx}, nHashType{hash_type}, amount{amount}, + checker{txdata ? MutableTransactionSignatureChecker{&m_txto, nIn, amount, *txdata, MissingDataBehavior::FAIL} : + MutableTransactionSignatureChecker{&m_txto, nIn, amount, MissingDataBehavior::FAIL}}, m_txdata(txdata) { } @@ -50,7 +50,7 @@ bool MutableTransactionSignatureCreator::CreateSig(const SigningProvider& provid // BASE/WITNESS_V0 signatures don't support explicit SIGHASH_DEFAULT, use SIGHASH_ALL instead. const int hashtype = nHashType == SIGHASH_DEFAULT ? SIGHASH_ALL : nHashType; - uint256 hash = SignatureHash(scriptCode, *txTo, nIn, hashtype, amount, sigversion, m_txdata); + uint256 hash = SignatureHash(scriptCode, m_txto, nIn, hashtype, amount, sigversion, m_txdata); if (!key.Sign(hash, vchSig)) return false; vchSig.push_back((unsigned char)hashtype); @@ -80,7 +80,7 @@ bool MutableTransactionSignatureCreator::CreateSchnorrSig(const SigningProvider& execdata.m_tapleaf_hash = *leaf_hash; } uint256 hash; - if (!SignatureHashSchnorr(hash, execdata, *txTo, nIn, nHashType, sigversion, *m_txdata, MissingDataBehavior::FAIL)) return false; + if (!SignatureHashSchnorr(hash, execdata, m_txto, nIn, nHashType, sigversion, *m_txdata, MissingDataBehavior::FAIL)) return false; sig.resize(64); // Use uint256{} as aux_rnd for now. if (!key.SignSchnorr(hash, sig, merkle_root, {})) return false; @@ -174,6 +174,29 @@ static bool SignTaprootScript(const SigningProvider& provider, const BaseSignatu result = Vector(std::move(sig)); return true; } + return false; + } + + // multi_a scripts (<key> OP_CHECKSIG <key> OP_CHECKSIGADD <key> OP_CHECKSIGADD <k> OP_NUMEQUAL) + if (auto match = MatchMultiA(script)) { + std::vector<std::vector<unsigned char>> sigs; + int good_sigs = 0; + for (size_t i = 0; i < match->second.size(); ++i) { + XOnlyPubKey pubkey{*(match->second.rbegin() + i)}; + std::vector<unsigned char> sig; + bool good_sig = CreateTaprootScriptSig(creator, sigdata, provider, sig, pubkey, leaf_hash, sigversion); + if (good_sig && good_sigs < match->first) { + ++good_sigs; + sigs.push_back(std::move(sig)); + } else { + sigs.emplace_back(); + } + } + if (good_sigs == match->first) { + result = std::move(sigs); + return true; + } + return false; } return false; @@ -517,7 +540,7 @@ bool SignSignature(const SigningProvider &provider, const CScript& fromPubKey, C { assert(nIn < txTo.vin.size()); - MutableTransactionSignatureCreator creator(&txTo, nIn, amount, nHashType); + MutableTransactionSignatureCreator creator(txTo, nIn, amount, nHashType); SignatureData sigdata; bool ret = ProduceSignature(provider, creator, fromPubKey, sigdata); @@ -632,7 +655,7 @@ bool SignTransaction(CMutableTransaction& mtx, const SigningProvider* keystore, CTxIn& txin = mtx.vin[i]; auto coin = coins.find(txin.prevout); if (coin == coins.end() || coin->second.IsSpent()) { - txdata.Init(txConst, /* spent_outputs */ {}, /* force */ true); + txdata.Init(txConst, /*spent_outputs=*/{}, /*force=*/true); break; } else { spent_outputs.emplace_back(coin->second.out.nValue, coin->second.out.scriptPubKey); @@ -656,7 +679,7 @@ bool SignTransaction(CMutableTransaction& mtx, const SigningProvider* keystore, SignatureData sigdata = DataFromTransaction(mtx, i, coin->second.out); // Only sign SIGHASH_SINGLE if there's a corresponding output: if (!fHashSingle || (i < mtx.vout.size())) { - ProduceSignature(*keystore, MutableTransactionSignatureCreator(&mtx, i, amount, &txdata, nHashType), prevPubKey, sigdata); + ProduceSignature(*keystore, MutableTransactionSignatureCreator(mtx, i, amount, &txdata, nHashType), prevPubKey, sigdata); } UpdateInput(txin, sigdata); diff --git a/src/script/sign.h b/src/script/sign.h index 7301826ba5..71203d08ec 100644 --- a/src/script/sign.h +++ b/src/script/sign.h @@ -6,6 +6,7 @@ #ifndef BITCOIN_SCRIPT_SIGN_H #define BITCOIN_SCRIPT_SIGN_H +#include <attributes.h> #include <coins.h> #include <hash.h> #include <pubkey.h> @@ -34,8 +35,9 @@ public: }; /** A signature creator for transactions. */ -class MutableTransactionSignatureCreator : public BaseSignatureCreator { - const CMutableTransaction* txTo; +class MutableTransactionSignatureCreator : public BaseSignatureCreator +{ + const CMutableTransaction& m_txto; unsigned int nIn; int nHashType; CAmount amount; @@ -43,8 +45,8 @@ class MutableTransactionSignatureCreator : public BaseSignatureCreator { const PrecomputedTransactionData* m_txdata; public: - MutableTransactionSignatureCreator(const CMutableTransaction* tx, unsigned int input_idx, const CAmount& amount, int hash_type); - MutableTransactionSignatureCreator(const CMutableTransaction* tx, unsigned int input_idx, const CAmount& amount, const PrecomputedTransactionData* txdata, int hash_type); + MutableTransactionSignatureCreator(const CMutableTransaction& tx LIFETIMEBOUND, unsigned int input_idx, const CAmount& amount, int hash_type); + MutableTransactionSignatureCreator(const CMutableTransaction& tx LIFETIMEBOUND, unsigned int input_idx, const CAmount& amount, const PrecomputedTransactionData* txdata, int hash_type); const BaseSignatureChecker& Checker() const override { return checker; } bool CreateSig(const SigningProvider& provider, std::vector<unsigned char>& vchSig, const CKeyID& keyid, const CScript& scriptCode, SigVersion sigversion) const override; bool CreateSchnorrSig(const SigningProvider& provider, std::vector<unsigned char>& sig, const XOnlyPubKey& pubkey, const uint256* leaf_hash, const uint256* merkle_root, SigVersion sigversion) const override; diff --git a/src/script/standard.cpp b/src/script/standard.cpp index 5fb98cc307..e25155d3dd 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -91,56 +91,83 @@ static constexpr bool IsSmallInteger(opcodetype opcode) return opcode >= OP_1 && opcode <= OP_16; } -static constexpr bool IsPushdataOp(opcodetype opcode) -{ - return opcode > OP_FALSE && opcode <= OP_PUSHDATA4; -} - -static constexpr bool IsValidMultisigKeyCount(int n_keys) -{ - return n_keys > 0 && n_keys <= MAX_PUBKEYS_PER_MULTISIG; -} - -static bool GetMultisigKeyCount(opcodetype opcode, valtype data, int& count) +/** Retrieve a minimally-encoded number in range [min,max] from an (opcode, data) pair, + * whether it's OP_n or through a push. */ +static std::optional<int> GetScriptNumber(opcodetype opcode, valtype data, int min, int max) { + int count; if (IsSmallInteger(opcode)) { count = CScript::DecodeOP_N(opcode); - return IsValidMultisigKeyCount(count); - } - - if (IsPushdataOp(opcode)) { - if (!CheckMinimalPush(data, opcode)) return false; + } else if (IsPushdataOp(opcode)) { + if (!CheckMinimalPush(data, opcode)) return {}; try { count = CScriptNum(data, /* fRequireMinimal = */ true).getint(); - return IsValidMultisigKeyCount(count); } catch (const scriptnum_error&) { - return false; + return {}; } + } else { + return {}; } - - return false; + if (count < min || count > max) return {}; + return count; } static bool MatchMultisig(const CScript& script, int& required_sigs, std::vector<valtype>& pubkeys) { opcodetype opcode; valtype data; - int num_keys; CScript::const_iterator it = script.begin(); if (script.size() < 1 || script.back() != OP_CHECKMULTISIG) return false; - if (!script.GetOp(it, opcode, data) || !GetMultisigKeyCount(opcode, data, required_sigs)) return false; + if (!script.GetOp(it, opcode, data)) return false; + auto req_sigs = GetScriptNumber(opcode, data, 1, MAX_PUBKEYS_PER_MULTISIG); + if (!req_sigs) return false; + required_sigs = *req_sigs; while (script.GetOp(it, opcode, data) && CPubKey::ValidSize(data)) { pubkeys.emplace_back(std::move(data)); } - if (!GetMultisigKeyCount(opcode, data, num_keys)) return false; - - if (pubkeys.size() != static_cast<unsigned long>(num_keys) || num_keys < required_sigs) return false; + auto num_keys = GetScriptNumber(opcode, data, required_sigs, MAX_PUBKEYS_PER_MULTISIG); + if (!num_keys) return false; + if (pubkeys.size() != static_cast<unsigned long>(*num_keys)) return false; return (it + 1 == script.end()); } +std::optional<std::pair<int, std::vector<Span<const unsigned char>>>> MatchMultiA(const CScript& script) +{ + std::vector<Span<const unsigned char>> keyspans; + + // Redundant, but very fast and selective test. + if (script.size() == 0 || script[0] != 32 || script.back() != OP_NUMEQUAL) return {}; + + // Parse keys + auto it = script.begin(); + while (script.end() - it >= 34) { + if (*it != 32) return {}; + ++it; + keyspans.emplace_back(&*it, 32); + it += 32; + if (*it != (keyspans.size() == 1 ? OP_CHECKSIG : OP_CHECKSIGADD)) return {}; + ++it; + } + if (keyspans.size() == 0 || keyspans.size() > MAX_PUBKEYS_PER_MULTI_A) return {}; + + // Parse threshold. + opcodetype opcode; + std::vector<unsigned char> data; + if (!script.GetOp(it, opcode, data)) return {}; + if (it == script.end()) return {}; + if (*it != OP_NUMEQUAL) return {}; + ++it; + if (it != script.end()) return {}; + auto threshold = GetScriptNumber(opcode, data, 1, (int)keyspans.size()); + if (!threshold) return {}; + + // Construct result. + return std::pair{*threshold, std::move(keyspans)}; +} + TxoutType Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned char>>& vSolutionsRet) { vSolutionsRet.clear(); @@ -366,13 +393,7 @@ void TaprootSpendData::Merge(TaprootSpendData other) merkle_root = other.merkle_root; } for (auto& [key, control_blocks] : other.scripts) { - // Once P0083R3 is supported by all our targeted platforms, - // this loop body can be replaced with: - // scripts[key].merge(std::move(control_blocks)); - auto& target = scripts[key]; - for (auto& control_block: control_blocks) { - target.insert(std::move(control_block)); - } + scripts[key].merge(std::move(control_blocks)); } } diff --git a/src/script/standard.h b/src/script/standard.h index eb50421768..f0b143c52b 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -162,6 +162,11 @@ bool IsValidDestination(const CTxDestination& dest); /** Get the name of a TxoutType as a string */ std::string GetTxnOutputType(TxoutType t); +constexpr bool IsPushdataOp(opcodetype opcode) +{ + return opcode > OP_FALSE && opcode <= OP_PUSHDATA4; +} + /** * Parse a scriptPubKey and identify script type for standard scripts. If * successful, returns script type and parsed pubkeys or hashes, depending on @@ -191,6 +196,10 @@ CScript GetScriptForDestination(const CTxDestination& dest); /** Generate a P2PK script for the given pubkey. */ CScript GetScriptForRawPubKey(const CPubKey& pubkey); +/** Determine if script is a "multi_a" script. Returns (threshold, keyspans) if so, and nullopt otherwise. + * The keyspans refer to bytes in the passed script. */ +std::optional<std::pair<int, std::vector<Span<const unsigned char>>>> MatchMultiA(const CScript& script LIFETIMEBOUND); + /** Generate a multisig script. */ CScript GetScriptForMultisig(int nRequired, const std::vector<CPubKey>& keys); diff --git a/src/secp256k1/.cirrus.yml b/src/secp256k1/.cirrus.yml index 35a9a45367..a2e7f36d1f 100644 --- a/src/secp256k1/.cirrus.yml +++ b/src/secp256k1/.cirrus.yml @@ -4,10 +4,10 @@ env: # Specific warnings can be disabled with -Wno-error=foo. # -pedantic-errors is not equivalent to -Werror=pedantic and thus not implied by -Werror according to the GCC manual. WERROR_CFLAGS: -Werror -pedantic-errors - MAKEFLAGS: -j2 + MAKEFLAGS: -j4 BUILD: check ### secp256k1 config - STATICPRECOMPUTATION: yes + ECMULTWINDOW: auto ECMULTGENPRECISION: auto ASM: no WIDEMUL: auto @@ -23,6 +23,8 @@ env: BENCH: yes SECP256K1_BENCH_ITERS: 2 CTIMETEST: yes + # Compile and run the tests + EXAMPLES: yes cat_logs_snippet: &CAT_LOGS always: @@ -50,28 +52,32 @@ merge_base_script_snippet: &MERGE_BASE - git config --global user.name "ci" - git merge FETCH_HEAD # Merge base to detect silent merge conflicts -task: - name: "x86_64: Linux (Debian stable)" +linux_container_snippet: &LINUX_CONTAINER container: dockerfile: ci/linux-debian.Dockerfile # Reduce number of CPUs to be able to do more builds in parallel. cpu: 1 + # Gives us more CPUs for free if they're available. + greedy: true # More than enough for our scripts. memory: 1G + +task: + name: "x86_64: Linux (Debian stable)" + << : *LINUX_CONTAINER matrix: &ENV_MATRIX - env: {WIDEMUL: int64, RECOVERY: yes} - - env: {WIDEMUL: int64, ECDH: yes, EXPERIMENTAL: yes, SCHNORRSIG: yes} + - env: {WIDEMUL: int64, ECDH: yes, SCHNORRSIG: yes} - env: {WIDEMUL: int128} - - env: {WIDEMUL: int128, RECOVERY: yes, EXPERIMENTAL: yes, SCHNORRSIG: yes} - - env: {WIDEMUL: int128, ECDH: yes, EXPERIMENTAL: yes, SCHNORRSIG: yes} + - env: {WIDEMUL: int128, RECOVERY: yes, SCHNORRSIG: yes} + - env: {WIDEMUL: int128, ECDH: yes, SCHNORRSIG: yes} - env: {WIDEMUL: int128, ASM: x86_64} - - env: { RECOVERY: yes, EXPERIMENTAL: yes, SCHNORRSIG: yes} - - env: { STATICPRECOMPUTATION: no} + - env: { RECOVERY: yes, SCHNORRSIG: yes} - env: {BUILD: distcheck, WITH_VALGRIND: no, CTIMETEST: no, BENCH: no} - env: {CPPFLAGS: -DDETERMINISTIC} - env: {CFLAGS: -O0, CTIMETEST: no} - - env: { ECMULTGENPRECISION: 2 } - - env: { ECMULTGENPRECISION: 8 } + - env: { ECMULTGENPRECISION: 2, ECMULTWINDOW: 2 } + - env: { ECMULTGENPRECISION: 8, ECMULTWINDOW: 4 } matrix: - env: CC: gcc @@ -84,15 +90,11 @@ task: task: name: "i686: Linux (Debian stable)" - container: - dockerfile: ci/linux-debian.Dockerfile - cpu: 1 - memory: 1G + << : *LINUX_CONTAINER env: HOST: i686-linux-gnu ECDH: yes RECOVERY: yes - EXPERIMENTAL: yes SCHNORRSIG: yes matrix: - env: @@ -134,8 +136,10 @@ task: ## - rm /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress ## brew_valgrind_pre_script: + # Retry a few times because this tends to fail randomly. + - for i in {1..5}; do brew update && break || sleep 15; done - brew config - - brew tap --shallow LouisBrunner/valgrind + - brew tap LouisBrunner/valgrind # Fetch valgrind source but don't build it yet. - brew fetch --HEAD LouisBrunner/valgrind/valgrind brew_valgrind_cache: @@ -165,10 +169,7 @@ task: task: name: "s390x (big-endian): Linux (Debian stable, QEMU)" - container: - dockerfile: ci/linux-debian.Dockerfile - cpu: 1 - memory: 1G + << : *LINUX_CONTAINER env: WRAPPER_CMD: qemu-s390x SECP256K1_TEST_ITERS: 16 @@ -176,7 +177,6 @@ task: WITH_VALGRIND: no ECDH: yes RECOVERY: yes - EXPERIMENTAL: yes SCHNORRSIG: yes CTIMETEST: no << : *MERGE_BASE @@ -188,10 +188,7 @@ task: task: name: "ARM32: Linux (Debian stable, QEMU)" - container: - dockerfile: ci/linux-debian.Dockerfile - cpu: 1 - memory: 1G + << : *LINUX_CONTAINER env: WRAPPER_CMD: qemu-arm SECP256K1_TEST_ITERS: 16 @@ -199,12 +196,11 @@ task: WITH_VALGRIND: no ECDH: yes RECOVERY: yes - EXPERIMENTAL: yes SCHNORRSIG: yes CTIMETEST: no matrix: - env: {} - - env: {ASM: arm} + - env: {EXPERIMENTAL: yes, ASM: arm} << : *MERGE_BASE test_script: - ./ci/cirrus.sh @@ -212,10 +208,7 @@ task: task: name: "ARM64: Linux (Debian stable, QEMU)" - container: - dockerfile: ci/linux-debian.Dockerfile - cpu: 1 - memory: 1G + << : *LINUX_CONTAINER env: WRAPPER_CMD: qemu-aarch64 SECP256K1_TEST_ITERS: 16 @@ -223,7 +216,6 @@ task: WITH_VALGRIND: no ECDH: yes RECOVERY: yes - EXPERIMENTAL: yes SCHNORRSIG: yes CTIMETEST: no << : *MERGE_BASE @@ -233,10 +225,7 @@ task: task: name: "ppc64le: Linux (Debian stable, QEMU)" - container: - dockerfile: ci/linux-debian.Dockerfile - cpu: 1 - memory: 1G + << : *LINUX_CONTAINER env: WRAPPER_CMD: qemu-ppc64le SECP256K1_TEST_ITERS: 16 @@ -244,7 +233,6 @@ task: WITH_VALGRIND: no ECDH: yes RECOVERY: yes - EXPERIMENTAL: yes SCHNORRSIG: yes CTIMETEST: no << : *MERGE_BASE @@ -254,10 +242,7 @@ task: task: name: "x86_64 (mingw32-w64): Windows (Debian stable, Wine)" - container: - dockerfile: ci/linux-debian.Dockerfile - cpu: 1 - memory: 1G + << : *LINUX_CONTAINER env: WRAPPER_CMD: wine64-stable SECP256K1_TEST_ITERS: 16 @@ -265,7 +250,6 @@ task: WITH_VALGRIND: no ECDH: yes RECOVERY: yes - EXPERIMENTAL: yes SCHNORRSIG: yes CTIMETEST: no << : *MERGE_BASE @@ -275,23 +259,23 @@ task: # Sanitizers task: - container: - dockerfile: ci/linux-debian.Dockerfile - cpu: 1 - memory: 2G + << : *LINUX_CONTAINER env: ECDH: yes RECOVERY: yes - EXPERIMENTAL: yes SCHNORRSIG: yes CTIMETEST: no matrix: - name: "Valgrind (memcheck)" + container: + cpu: 2 env: # The `--error-exitcode` is required to make the test fail if valgrind found errors, otherwise it'll return 0 (https://www.valgrind.org/docs/manual/manual-core.html) WRAPPER_CMD: "valgrind --error-exitcode=42" SECP256K1_TEST_ITERS: 2 - name: "UBSan, ASan, LSan" + container: + memory: 2G env: CFLAGS: "-fsanitize=undefined,address -g" UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1" @@ -302,11 +286,10 @@ task: matrix: - env: ASM: auto - STATICPRECOMPUTATION: yes - env: ASM: no - STATICPRECOMPUTATION: no ECMULTGENPRECISION: 2 + ECMULTWINDOW: 2 matrix: - env: CC: clang @@ -320,17 +303,13 @@ task: task: name: "C++ -fpermissive" - container: - dockerfile: ci/linux-debian.Dockerfile - cpu: 1 - memory: 1G + << : *LINUX_CONTAINER env: # ./configure correctly errors out when given CC=g++. # We hack around this by passing CC=g++ only to make. CC: gcc - MAKEFLAGS: -j2 CC=g++ CFLAGS=-fpermissive\ -g + MAKEFLAGS: -j4 CC=g++ CFLAGS=-fpermissive\ -g WERROR_CFLAGS: - EXPERIMENTAL: yes ECDH: yes RECOVERY: yes SCHNORRSIG: yes @@ -338,3 +317,10 @@ task: test_script: - ./ci/cirrus.sh << : *CAT_LOGS + +task: + name: "sage prover" + << : *LINUX_CONTAINER + test_script: + - cd sage + - sage prove_group_implementations.sage diff --git a/src/secp256k1/.gitattributes b/src/secp256k1/.gitattributes index a0fa567da8..30efb2244f 100644 --- a/src/secp256k1/.gitattributes +++ b/src/secp256k1/.gitattributes @@ -1,2 +1,2 @@ -src/ecmult_static_pre_g.h linguist-generated -src/ecmult_gen_static_prec_table.h linguist-generated +src/precomputed_ecmult.c linguist-generated +src/precomputed_ecmult_gen.c linguist-generated diff --git a/src/secp256k1/.gitignore b/src/secp256k1/.gitignore index 22cd500501..d88627d72e 100644 --- a/src/secp256k1/.gitignore +++ b/src/secp256k1/.gitignore @@ -3,14 +3,19 @@ bench_ecmult bench_internal tests exhaustive_tests -gen_ecmult_gen_static_prec_table -gen_ecmult_static_pre_g +precompute_ecmult_gen +precompute_ecmult valgrind_ctime_test +ecdh_example +ecdsa_example +schnorr_example *.exe *.so *.a *.csv !.gitignore +*.log +*.trs Makefile configure @@ -41,6 +46,7 @@ coverage.*.html src/libsecp256k1-config.h src/libsecp256k1-config.h.in +build-aux/ar-lib build-aux/config.guess build-aux/config.sub build-aux/depcomp diff --git a/src/secp256k1/Makefile.am b/src/secp256k1/Makefile.am index 7ea29bc6e3..51c5960301 100644 --- a/src/secp256k1/Makefile.am +++ b/src/secp256k1/Makefile.am @@ -26,12 +26,14 @@ noinst_HEADERS += src/eckey.h noinst_HEADERS += src/eckey_impl.h noinst_HEADERS += src/ecmult.h noinst_HEADERS += src/ecmult_impl.h +noinst_HEADERS += src/ecmult_compute_table.h +noinst_HEADERS += src/ecmult_compute_table_impl.h noinst_HEADERS += src/ecmult_const.h noinst_HEADERS += src/ecmult_const_impl.h noinst_HEADERS += src/ecmult_gen.h noinst_HEADERS += src/ecmult_gen_impl.h -noinst_HEADERS += src/ecmult_gen_prec.h -noinst_HEADERS += src/ecmult_gen_prec_impl.h +noinst_HEADERS += src/ecmult_gen_compute_table.h +noinst_HEADERS += src/ecmult_gen_compute_table_impl.h noinst_HEADERS += src/field_10x26.h noinst_HEADERS += src/field_10x26_impl.h noinst_HEADERS += src/field_5x52.h @@ -42,6 +44,8 @@ noinst_HEADERS += src/modinv32.h noinst_HEADERS += src/modinv32_impl.h noinst_HEADERS += src/modinv64.h noinst_HEADERS += src/modinv64_impl.h +noinst_HEADERS += src/precomputed_ecmult.h +noinst_HEADERS += src/precomputed_ecmult_gen.h noinst_HEADERS += src/assumptions.h noinst_HEADERS += src/util.h noinst_HEADERS += src/scratch.h @@ -59,13 +63,19 @@ noinst_HEADERS += contrib/lax_der_parsing.h noinst_HEADERS += contrib/lax_der_parsing.c noinst_HEADERS += contrib/lax_der_privatekey_parsing.h noinst_HEADERS += contrib/lax_der_privatekey_parsing.c +noinst_HEADERS += examples/random.h + +PRECOMPUTED_LIB = libsecp256k1_precomputed.la +noinst_LTLIBRARIES = $(PRECOMPUTED_LIB) +libsecp256k1_precomputed_la_SOURCES = src/precomputed_ecmult.c src/precomputed_ecmult_gen.c +libsecp256k1_precomputed_la_CPPFLAGS = $(SECP_INCLUDES) if USE_EXTERNAL_ASM COMMON_LIB = libsecp256k1_common.la -noinst_LTLIBRARIES = $(COMMON_LIB) else COMMON_LIB = endif +noinst_LTLIBRARIES += $(COMMON_LIB) pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = libsecp256k1.pc @@ -78,8 +88,8 @@ endif libsecp256k1_la_SOURCES = src/secp256k1.c libsecp256k1_la_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/src $(SECP_INCLUDES) -libsecp256k1_la_LIBADD = $(SECP_LIBS) $(COMMON_LIB) -libsecp256k1_la_LDFLAGS = -no-undefined +libsecp256k1_la_LIBADD = $(SECP_LIBS) $(COMMON_LIB) $(PRECOMPUTED_LIB) +libsecp256k1_la_LDFLAGS = -no-undefined -version-info $(LIB_VERSION_CURRENT):$(LIB_VERSION_REVISION):$(LIB_VERSION_AGE) if VALGRIND_ENABLED libsecp256k1_la_CPPFLAGS += -DVALGRIND @@ -91,10 +101,10 @@ noinst_PROGRAMS += bench bench_internal bench_ecmult bench_SOURCES = src/bench.c bench_LDADD = libsecp256k1.la $(SECP_LIBS) $(SECP_TEST_LIBS) $(COMMON_LIB) bench_internal_SOURCES = src/bench_internal.c -bench_internal_LDADD = $(SECP_LIBS) $(COMMON_LIB) +bench_internal_LDADD = $(SECP_LIBS) $(COMMON_LIB) $(PRECOMPUTED_LIB) bench_internal_CPPFLAGS = $(SECP_INCLUDES) bench_ecmult_SOURCES = src/bench_ecmult.c -bench_ecmult_LDADD = $(SECP_LIBS) $(COMMON_LIB) +bench_ecmult_LDADD = $(SECP_LIBS) $(COMMON_LIB) $(PRECOMPUTED_LIB) bench_ecmult_CPPFLAGS = $(SECP_INCLUDES) endif @@ -112,7 +122,7 @@ endif if !ENABLE_COVERAGE tests_CPPFLAGS += -DVERIFY endif -tests_LDADD = $(SECP_LIBS) $(SECP_TEST_LIBS) $(COMMON_LIB) +tests_LDADD = $(SECP_LIBS) $(SECP_TEST_LIBS) $(COMMON_LIB) $(PRECOMPUTED_LIB) tests_LDFLAGS = -static TESTS += tests endif @@ -124,22 +134,57 @@ exhaustive_tests_CPPFLAGS = $(SECP_INCLUDES) if !ENABLE_COVERAGE exhaustive_tests_CPPFLAGS += -DVERIFY endif +# Note: do not include $(PRECOMPUTED_LIB) in exhaustive_tests (it uses runtime-generated tables). exhaustive_tests_LDADD = $(SECP_LIBS) $(COMMON_LIB) exhaustive_tests_LDFLAGS = -static TESTS += exhaustive_tests endif +if USE_EXAMPLES +noinst_PROGRAMS += ecdsa_example +ecdsa_example_SOURCES = examples/ecdsa.c +ecdsa_example_CPPFLAGS = -I$(top_srcdir)/include +ecdsa_example_LDADD = libsecp256k1.la +ecdsa_example_LDFLAGS = -static +if BUILD_WINDOWS +ecdsa_example_LDFLAGS += -lbcrypt +endif +TESTS += ecdsa_example +if ENABLE_MODULE_ECDH +noinst_PROGRAMS += ecdh_example +ecdh_example_SOURCES = examples/ecdh.c +ecdh_example_CPPFLAGS = -I$(top_srcdir)/include +ecdh_example_LDADD = libsecp256k1.la +ecdh_example_LDFLAGS = -static +if BUILD_WINDOWS +ecdh_example_LDFLAGS += -lbcrypt +endif +TESTS += ecdh_example +endif +if ENABLE_MODULE_SCHNORRSIG +noinst_PROGRAMS += schnorr_example +schnorr_example_SOURCES = examples/schnorr.c +schnorr_example_CPPFLAGS = -I$(top_srcdir)/include +schnorr_example_LDADD = libsecp256k1.la +schnorr_example_LDFLAGS = -static +if BUILD_WINDOWS +schnorr_example_LDFLAGS += -lbcrypt +endif +TESTS += schnorr_example +endif +endif + ### Precomputed tables -EXTRA_PROGRAMS = gen_ecmult_static_pre_g gen_ecmult_gen_static_prec_table +EXTRA_PROGRAMS = precompute_ecmult precompute_ecmult_gen CLEANFILES = $(EXTRA_PROGRAMS) -gen_ecmult_static_pre_g_SOURCES = src/gen_ecmult_static_pre_g.c -gen_ecmult_static_pre_g_CPPFLAGS = $(SECP_INCLUDES) -gen_ecmult_static_pre_g_LDADD = $(SECP_LIBS) $(COMMON_LIB) +precompute_ecmult_SOURCES = src/precompute_ecmult.c +precompute_ecmult_CPPFLAGS = $(SECP_INCLUDES) +precompute_ecmult_LDADD = $(SECP_LIBS) $(COMMON_LIB) -gen_ecmult_gen_static_prec_table_SOURCES = src/gen_ecmult_gen_static_prec_table.c -gen_ecmult_gen_static_prec_table_CPPFLAGS = $(SECP_INCLUDES) -gen_ecmult_gen_static_prec_table_LDADD = $(SECP_LIBS) $(COMMON_LIB) +precompute_ecmult_gen_SOURCES = src/precompute_ecmult_gen.c +precompute_ecmult_gen_CPPFLAGS = $(SECP_INCLUDES) +precompute_ecmult_gen_LDADD = $(SECP_LIBS) $(COMMON_LIB) # See Automake manual, Section "Errors with distclean". # We don't list any dependencies for the prebuilt files here because @@ -147,15 +192,14 @@ gen_ecmult_gen_static_prec_table_LDADD = $(SECP_LIBS) $(COMMON_LIB) # build by a normal user) depends on mtimes, and thus is very fragile. # This means that rebuilds of the prebuilt files always need to be # forced by deleting them, e.g., by invoking `make clean-precomp`. -src/ecmult_static_pre_g.h: - $(MAKE) $(AM_MAKEFLAGS) gen_ecmult_static_pre_g$(EXEEXT) - ./gen_ecmult_static_pre_g$(EXEEXT) -src/ecmult_gen_static_prec_table.h: - $(MAKE) $(AM_MAKEFLAGS) gen_ecmult_gen_static_prec_table$(EXEEXT) - ./gen_ecmult_gen_static_prec_table$(EXEEXT) - -PRECOMP = src/ecmult_gen_static_prec_table.h src/ecmult_static_pre_g.h -noinst_HEADERS += $(PRECOMP) +src/precomputed_ecmult.c: + $(MAKE) $(AM_MAKEFLAGS) precompute_ecmult$(EXEEXT) + ./precompute_ecmult$(EXEEXT) +src/precomputed_ecmult_gen.c: + $(MAKE) $(AM_MAKEFLAGS) precompute_ecmult_gen$(EXEEXT) + ./precompute_ecmult_gen$(EXEEXT) + +PRECOMP = src/precomputed_ecmult_gen.c src/precomputed_ecmult.c precomp: $(PRECOMP) # Ensure the prebuilt files will be build first (only if they don't exist, diff --git a/src/secp256k1/README.md b/src/secp256k1/README.md index 5fc07dd4fa..f5db915e83 100644 --- a/src/secp256k1/README.md +++ b/src/secp256k1/README.md @@ -17,9 +17,7 @@ Features: * Suitable for embedded systems. * Optional module for public key recovery. * Optional module for ECDH key exchange. -* Optional module for Schnorr signatures according to [BIP-340](https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki) (experimental). - -Experimental features have not received enough scrutiny to satisfy the standard of quality of this library but are made available for testing and review by the community. The APIs of these features should not be considered stable. +* Optional module for Schnorr signatures according to [BIP-340](https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki). Implementation details ---------------------- @@ -35,6 +33,7 @@ Implementation details * Optimized implementation of arithmetic modulo the curve's field size (2^256 - 0x1000003D1). * Using 5 52-bit limbs (including hand-optimized assembly for x86_64, by Diederik Huys). * Using 10 26-bit limbs (including hand-optimized assembly for 32-bit ARM, by Wladimir J. van der Laan). + * This is an experimental feature that has not received enough scrutiny to satisfy the standard of quality of this library but is made available for testing and review by the community. * Scalar operations * Optimized implementation without data-dependent branches of arithmetic modulo the curve's order. * Using 4 64-bit limbs (relying on __int128 support in the compiler). @@ -69,6 +68,16 @@ libsecp256k1 is built using autotools: $ make check # run the test suite $ sudo make install # optional +To compile optional modules (such as Schnorr signatures), you need to run `./configure` with additional flags (such as `--enable-module-schnorrsig`). Run `./configure --help` to see the full list of available flags. + +Usage examples +----------- + Usage examples can be found in the [examples](examples) directory. To compile them you need to configure with `--enable-examples`. + * [ECDSA example](examples/ecdsa.c) + * [Schnorr signatures example](examples/schnorr.c) + * [Deriving a shared secret (ECDH) example](examples/ecdh.c) + To compile the Schnorr signature and ECDH examples, you also need to configure with `--enable-module-schnorrsig` and `--enable-module-ecdh`. + Test coverage ----------- diff --git a/src/secp256k1/build-aux/m4/bitcoin_secp.m4 b/src/secp256k1/build-aux/m4/bitcoin_secp.m4 index c14d09fa1b..dda770e001 100644 --- a/src/secp256k1/build-aux/m4/bitcoin_secp.m4 +++ b/src/secp256k1/build-aux/m4/bitcoin_secp.m4 @@ -38,3 +38,16 @@ AC_DEFUN([SECP_TRY_APPEND_CFLAGS], [ unset flag_works AC_SUBST($2) ]) + +dnl SECP_SET_DEFAULT(VAR, default, default-dev-mode) +dnl Set VAR to default or default-dev-mode, depending on whether dev mode is enabled +AC_DEFUN([SECP_SET_DEFAULT], [ + if test "${enable_dev_mode+set}" != set; then + AC_MSG_ERROR([[Set enable_dev_mode before calling SECP_SET_DEFAULT]]) + fi + if test x"$enable_dev_mode" = x"yes"; then + $1="$3" + else + $1="$2" + fi +]) diff --git a/src/secp256k1/ci/cirrus.sh b/src/secp256k1/ci/cirrus.sh index e27b34782e..b85f012d3f 100755 --- a/src/secp256k1/ci/cirrus.sh +++ b/src/secp256k1/ci/cirrus.sh @@ -15,9 +15,11 @@ valgrind --version || true ./configure \ --enable-experimental="$EXPERIMENTAL" \ --with-test-override-wide-multiply="$WIDEMUL" --with-asm="$ASM" \ - --enable-ecmult-static-precomputation="$STATICPRECOMPUTATION" --with-ecmult-gen-precision="$ECMULTGENPRECISION" \ + --with-ecmult-window="$ECMULTWINDOW" \ + --with-ecmult-gen-precision="$ECMULTGENPRECISION" \ --enable-module-ecdh="$ECDH" --enable-module-recovery="$RECOVERY" \ --enable-module-schnorrsig="$SCHNORRSIG" \ + --enable-examples="$EXAMPLES" \ --with-valgrind="$WITH_VALGRIND" \ --host="$HOST" $EXTRAFLAGS diff --git a/src/secp256k1/ci/linux-debian.Dockerfile b/src/secp256k1/ci/linux-debian.Dockerfile index fdba12aa00..5cccbb5565 100644 --- a/src/secp256k1/ci/linux-debian.Dockerfile +++ b/src/secp256k1/ci/linux-debian.Dockerfile @@ -19,7 +19,8 @@ RUN apt-get install --no-install-recommends --no-upgrade -y \ gcc-arm-linux-gnueabihf libc6-dev-armhf-cross libc6-dbg:armhf \ gcc-aarch64-linux-gnu libc6-dev-arm64-cross libc6-dbg:arm64 \ gcc-powerpc64le-linux-gnu libc6-dev-ppc64el-cross libc6-dbg:ppc64el \ - wine gcc-mingw-w64-x86-64 + wine gcc-mingw-w64-x86-64 \ + sagemath # Run a dummy command in wine to make it set up configuration RUN wine64-stable xcopy || true diff --git a/src/secp256k1/configure.ac b/src/secp256k1/configure.ac index 94feea7bb7..2db59a8ff3 100644 --- a/src/secp256k1/configure.ac +++ b/src/secp256k1/configure.ac @@ -1,30 +1,47 @@ AC_PREREQ([2.60]) -AC_INIT([libsecp256k1],[0.1]) + +# The package (a.k.a. release) version is based on semantic versioning 2.0.0 of +# the API. All changes in experimental modules are treated as +# backwards-compatible and therefore at most increase the minor version. +define(_PKG_VERSION_MAJOR, 0) +define(_PKG_VERSION_MINOR, 1) +define(_PKG_VERSION_BUILD, 0) +define(_PKG_VERSION_IS_RELEASE, false) + +# The library version is based on libtool versioning of the ABI. The set of +# rules for updating the version can be found here: +# https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html +# All changes in experimental modules are treated as if they don't affect the +# interface and therefore only increase the revision. +define(_LIB_VERSION_CURRENT, 0) +define(_LIB_VERSION_REVISION, 0) +define(_LIB_VERSION_AGE, 0) + +AC_INIT([libsecp256k1],m4_join([.], _PKG_VERSION_MAJOR, _PKG_VERSION_MINOR, _PKG_VERSION_BUILD)m4_if(_PKG_VERSION_IS_RELEASE, [true], [], [-pre]),[https://github.com/bitcoin-core/secp256k1/issues],[libsecp256k1],[https://github.com/bitcoin-core/secp256k1]) + AC_CONFIG_AUX_DIR([build-aux]) AC_CONFIG_MACRO_DIR([build-aux/m4]) AC_CANONICAL_HOST AH_TOP([#ifndef LIBSECP256K1_CONFIG_H]) AH_TOP([#define LIBSECP256K1_CONFIG_H]) AH_BOTTOM([#endif /*LIBSECP256K1_CONFIG_H*/]) -AM_INIT_AUTOMAKE([foreign subdir-objects]) -LT_INIT([win32-dll]) +# Require Automake 1.11.2 for AM_PROG_AR +AM_INIT_AUTOMAKE([1.11.2 foreign subdir-objects]) # Make the compilation flags quiet unless V=1 is used. m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) -PKG_PROG_PKG_CONFIG - -AC_PATH_TOOL(AR, ar) -AC_PATH_TOOL(RANLIB, ranlib) -AC_PATH_TOOL(STRIP, strip) - -AM_PROG_CC_C_O -AC_PROG_CC_C89 +AC_PROG_CC if test x"$ac_cv_prog_cc_c89" = x"no"; then AC_MSG_ERROR([c89 compiler support required]) fi AM_PROG_AS +AM_PROG_AR + +LT_INIT([win32-dll]) + +build_windows=no case $host_os in *darwin*) @@ -49,6 +66,9 @@ case $host_os in fi fi ;; + cygwin*|mingw*) + build_windows=yes + ;; esac # Try if some desirable compiler flags are supported and append them to SECP_CFLAGS. @@ -91,55 +111,54 @@ SECP_TRY_APPEND_DEFAULT_CFLAGS(SECP_CFLAGS) ### Define config arguments ### +# In dev mode, we enable all binaries and modules by default but individual options can still be overridden explicitly. +# Check for dev mode first because SECP_SET_DEFAULT needs enable_dev_mode set. +AC_ARG_ENABLE(dev_mode, [], [], + [enable_dev_mode=no]) + AC_ARG_ENABLE(benchmark, - AS_HELP_STRING([--enable-benchmark],[compile benchmark [default=yes]]), - [use_benchmark=$enableval], - [use_benchmark=yes]) + AS_HELP_STRING([--enable-benchmark],[compile benchmark [default=yes]]), [], + [SECP_SET_DEFAULT([enable_benchmark], [yes], [yes])]) AC_ARG_ENABLE(coverage, - AS_HELP_STRING([--enable-coverage],[enable compiler flags to support kcov coverage analysis [default=no]]), - [enable_coverage=$enableval], - [enable_coverage=no]) + AS_HELP_STRING([--enable-coverage],[enable compiler flags to support kcov coverage analysis [default=no]]), [], + [SECP_SET_DEFAULT([enable_coverage], [no], [no])]) AC_ARG_ENABLE(tests, - AS_HELP_STRING([--enable-tests],[compile tests [default=yes]]), - [use_tests=$enableval], - [use_tests=yes]) + AS_HELP_STRING([--enable-tests],[compile tests [default=yes]]), [], + [SECP_SET_DEFAULT([enable_tests], [yes], [yes])]) AC_ARG_ENABLE(experimental, - AS_HELP_STRING([--enable-experimental],[allow experimental configure options [default=no]]), - [use_experimental=$enableval], - [use_experimental=no]) + AS_HELP_STRING([--enable-experimental],[allow experimental configure options [default=no]]), [], + [SECP_SET_DEFAULT([enable_experimental], [no], [yes])]) AC_ARG_ENABLE(exhaustive_tests, - AS_HELP_STRING([--enable-exhaustive-tests],[compile exhaustive tests [default=yes]]), - [use_exhaustive_tests=$enableval], - [use_exhaustive_tests=yes]) + AS_HELP_STRING([--enable-exhaustive-tests],[compile exhaustive tests [default=yes]]), [], + [SECP_SET_DEFAULT([enable_exhaustive_tests], [yes], [yes])]) + +AC_ARG_ENABLE(examples, + AS_HELP_STRING([--enable-examples],[compile the examples [default=no]]), [], + [SECP_SET_DEFAULT([enable_examples], [no], [yes])]) AC_ARG_ENABLE(module_ecdh, - AS_HELP_STRING([--enable-module-ecdh],[enable ECDH shared secret computation]), - [enable_module_ecdh=$enableval], - [enable_module_ecdh=no]) + AS_HELP_STRING([--enable-module-ecdh],[enable ECDH module [default=no]]), [], + [SECP_SET_DEFAULT([enable_module_ecdh], [no], [yes])]) AC_ARG_ENABLE(module_recovery, - AS_HELP_STRING([--enable-module-recovery],[enable ECDSA pubkey recovery module [default=no]]), - [enable_module_recovery=$enableval], - [enable_module_recovery=no]) + AS_HELP_STRING([--enable-module-recovery],[enable ECDSA pubkey recovery module [default=no]]), [], + [SECP_SET_DEFAULT([enable_module_recovery], [no], [yes])]) AC_ARG_ENABLE(module_extrakeys, - AS_HELP_STRING([--enable-module-extrakeys],[enable extrakeys module (experimental)]), - [enable_module_extrakeys=$enableval], - [enable_module_extrakeys=no]) + AS_HELP_STRING([--enable-module-extrakeys],[enable extrakeys module [default=no]]), [], + [SECP_SET_DEFAULT([enable_module_extrakeys], [no], [yes])]) AC_ARG_ENABLE(module_schnorrsig, - AS_HELP_STRING([--enable-module-schnorrsig],[enable schnorrsig module (experimental)]), - [enable_module_schnorrsig=$enableval], - [enable_module_schnorrsig=no]) + AS_HELP_STRING([--enable-module-schnorrsig],[enable schnorrsig module [default=no]]), [], + [SECP_SET_DEFAULT([enable_module_schnorrsig], [no], [yes])]) AC_ARG_ENABLE(external_default_callbacks, - AS_HELP_STRING([--enable-external-default-callbacks],[enable external default callback functions [default=no]]), - [use_external_default_callbacks=$enableval], - [use_external_default_callbacks=no]) + AS_HELP_STRING([--enable-external-default-callbacks],[enable external default callback functions [default=no]]), [], + [SECP_SET_DEFAULT([enable_external_default_callbacks], [no], [no])]) # Test-only override of the (autodetected by the C code) "widemul" setting. # Legal values are int64 (for [u]int64_t), int128 (for [unsigned] __int128), and auto (the default). @@ -152,7 +171,7 @@ AC_ARG_WITH([ecmult-window], [AS_HELP_STRING([--with-ecmult-window=SIZE|auto], [window size for ecmult precomputation for verification, specified as integer in range [2..24].] [Larger values result in possibly better performance at the cost of an exponentially larger precomputed table.] [The table will store 2^(SIZE-1) * 64 bytes of data but can be larger in memory due to platform-specific padding and alignment.] -[A window size larger than 15 will require you delete the prebuilt ecmult_static_pre_g.h file so that it can be rebuilt.] +[A window size larger than 15 will require you delete the prebuilt precomputed_ecmult.c file so that it can be rebuilt.] [For very large window sizes, use "make -j 1" to reduce memory use during compilation.] ["auto" is a reasonable setting for desktop machines (currently 15). [default=auto]] )], @@ -229,14 +248,14 @@ else fi # Select assembly optimization -use_external_asm=no +enable_external_asm=no case $set_asm in x86_64) AC_DEFINE(USE_ASM_X86_64, 1, [Define this symbol to enable x86_64 assembly optimizations]) ;; arm) - use_external_asm=yes + enable_external_asm=yes ;; no) ;; @@ -245,7 +264,7 @@ no) ;; esac -if test x"$use_external_asm" = x"yes"; then +if test x"$enable_external_asm" = x"yes"; then AC_DEFINE(USE_EXTERNAL_ASM, 1, [Define this symbol if an external (non-inline) assembly implementation is used]) fi @@ -333,7 +352,7 @@ if test x"$enable_module_extrakeys" = x"yes"; then AC_DEFINE(ENABLE_MODULE_EXTRAKEYS, 1, [Define this symbol to enable the extrakeys module]) fi -if test x"$use_external_default_callbacks" = x"yes"; then +if test x"$enable_external_default_callbacks" = x"yes"; then AC_DEFINE(USE_EXTERNAL_DEFAULT_CALLBACKS, 1, [Define this symbol if an external implementation of the default callbacks is used]) fi @@ -345,16 +364,8 @@ if test x"$enable_experimental" = x"yes"; then AC_MSG_NOTICE([******]) AC_MSG_NOTICE([WARNING: experimental build]) AC_MSG_NOTICE([Experimental features do not have stable APIs or properties, and may not be safe for production use.]) - AC_MSG_NOTICE([Building extrakeys module: $enable_module_extrakeys]) - AC_MSG_NOTICE([Building schnorrsig module: $enable_module_schnorrsig]) AC_MSG_NOTICE([******]) else - if test x"$enable_module_extrakeys" = x"yes"; then - AC_MSG_ERROR([extrakeys module is experimental. Use --enable-experimental to allow.]) - fi - if test x"$enable_module_schnorrsig" = x"yes"; then - AC_MSG_ERROR([schnorrsig module is experimental. Use --enable-experimental to allow.]) - fi if test x"$set_asm" = x"arm"; then AC_MSG_ERROR([ARM assembly optimization is experimental. Use --enable-experimental to allow.]) fi @@ -372,29 +383,30 @@ AC_SUBST(SECP_TEST_LIBS) AC_SUBST(SECP_TEST_INCLUDES) AC_SUBST(SECP_CFLAGS) AM_CONDITIONAL([ENABLE_COVERAGE], [test x"$enable_coverage" = x"yes"]) -AM_CONDITIONAL([USE_TESTS], [test x"$use_tests" != x"no"]) -AM_CONDITIONAL([USE_EXHAUSTIVE_TESTS], [test x"$use_exhaustive_tests" != x"no"]) -AM_CONDITIONAL([USE_BENCHMARK], [test x"$use_benchmark" = x"yes"]) +AM_CONDITIONAL([USE_TESTS], [test x"$enable_tests" != x"no"]) +AM_CONDITIONAL([USE_EXHAUSTIVE_TESTS], [test x"$enable_exhaustive_tests" != x"no"]) +AM_CONDITIONAL([USE_EXAMPLES], [test x"$enable_examples" != x"no"]) +AM_CONDITIONAL([USE_BENCHMARK], [test x"$enable_benchmark" = x"yes"]) AM_CONDITIONAL([ENABLE_MODULE_ECDH], [test x"$enable_module_ecdh" = x"yes"]) AM_CONDITIONAL([ENABLE_MODULE_RECOVERY], [test x"$enable_module_recovery" = x"yes"]) AM_CONDITIONAL([ENABLE_MODULE_EXTRAKEYS], [test x"$enable_module_extrakeys" = x"yes"]) AM_CONDITIONAL([ENABLE_MODULE_SCHNORRSIG], [test x"$enable_module_schnorrsig" = x"yes"]) -AM_CONDITIONAL([USE_EXTERNAL_ASM], [test x"$use_external_asm" = x"yes"]) +AM_CONDITIONAL([USE_EXTERNAL_ASM], [test x"$enable_external_asm" = x"yes"]) AM_CONDITIONAL([USE_ASM_ARM], [test x"$set_asm" = x"arm"]) - -# Make sure nothing new is exported so that we don't break the cache. -PKGCONFIG_PATH_TEMP="$PKG_CONFIG_PATH" -unset PKG_CONFIG_PATH -PKG_CONFIG_PATH="$PKGCONFIG_PATH_TEMP" +AM_CONDITIONAL([BUILD_WINDOWS], [test "$build_windows" = "yes"]) +AC_SUBST(LIB_VERSION_CURRENT, _LIB_VERSION_CURRENT) +AC_SUBST(LIB_VERSION_REVISION, _LIB_VERSION_REVISION) +AC_SUBST(LIB_VERSION_AGE, _LIB_VERSION_AGE) AC_OUTPUT echo echo "Build Options:" -echo " with external callbacks = $use_external_default_callbacks" -echo " with benchmarks = $use_benchmark" -echo " with tests = $use_tests" +echo " with external callbacks = $enable_external_default_callbacks" +echo " with benchmarks = $enable_benchmark" +echo " with tests = $enable_tests" echo " with coverage = $enable_coverage" +echo " with examples = $enable_examples" echo " module ecdh = $enable_module_ecdh" echo " module recovery = $enable_module_recovery" echo " module extrakeys = $enable_module_extrakeys" diff --git a/src/secp256k1/doc/CHANGELOG.md b/src/secp256k1/doc/CHANGELOG.md new file mode 100644 index 0000000000..3c4c2e4583 --- /dev/null +++ b/src/secp256k1/doc/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +This file is currently only a template for future use. + +Each change falls into one of the following categories: Added, Changed, Deprecated, Removed, Fixed or Security. + +## [Unreleased] + +## [MAJOR.MINOR.PATCH] - YYYY-MM-DD + +### Added/Changed/Deprecated/Removed/Fixed/Security +- [Title with link to Pull Request](https://link-to-pr) diff --git a/src/secp256k1/doc/release-process.md b/src/secp256k1/doc/release-process.md new file mode 100644 index 0000000000..a35b8a9db3 --- /dev/null +++ b/src/secp256k1/doc/release-process.md @@ -0,0 +1,14 @@ +# Release Process + +1. Open PR to master that + 1. adds release notes to `doc/CHANGELOG.md` and + 2. if this is **not** a patch release, updates `_PKG_VERSION_{MAJOR,MINOR}` and `_LIB_VERSIONS_*` in `configure.ac` +2. After the PR is merged, + * if this is **not** a patch release, create a release branch with name `MAJOR.MINOR`. + Make sure that the branch contains the right commits. + Create commit on the release branch that sets `_PKG_VERSION_IS_RELEASE` in `configure.ac` to `true`. + * if this **is** a patch release, open a pull request with the bugfixes to the `MAJOR.MINOR` branch. + Also include the release note commit bump `_PKG_VERSION_BUILD` and `_LIB_VERSIONS_*` in `configure.ac`. +4. Tag the commit with `git tag -s vMAJOR.MINOR.PATCH`. +5. Push branch and tag with `git push origin --tags`. +6. Create a new GitHub release with a link to the corresponding entry in `doc/CHANGELOG.md`. diff --git a/src/secp256k1/examples/EXAMPLES_COPYING b/src/secp256k1/examples/EXAMPLES_COPYING new file mode 100644 index 0000000000..0e259d42c9 --- /dev/null +++ b/src/secp256k1/examples/EXAMPLES_COPYING @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/src/secp256k1/examples/ecdh.c b/src/secp256k1/examples/ecdh.c new file mode 100644 index 0000000000..d7e8add361 --- /dev/null +++ b/src/secp256k1/examples/ecdh.c @@ -0,0 +1,127 @@ +/************************************************************************* + * Written in 2020-2022 by Elichai Turkel * + * To the extent possible under law, the author(s) have dedicated all * + * copyright and related and neighboring rights to the software in this * + * file to the public domain worldwide. This software is distributed * + * without any warranty. For the CC0 Public Domain Dedication, see * + * EXAMPLES_COPYING or https://creativecommons.org/publicdomain/zero/1.0 * + *************************************************************************/ + +#include <stdio.h> +#include <assert.h> +#include <string.h> + +#include <secp256k1.h> +#include <secp256k1_ecdh.h> + +#include "random.h" + + +int main(void) { + unsigned char seckey1[32]; + unsigned char seckey2[32]; + unsigned char compressed_pubkey1[33]; + unsigned char compressed_pubkey2[33]; + unsigned char shared_secret1[32]; + unsigned char shared_secret2[32]; + unsigned char randomize[32]; + int return_val; + size_t len; + secp256k1_pubkey pubkey1; + secp256k1_pubkey pubkey2; + + /* The specification in secp256k1.h states that `secp256k1_ec_pubkey_create` + * needs a context object initialized for signing, which is why we create + * a context with the SECP256K1_CONTEXT_SIGN flag. + * (The docs for `secp256k1_ecdh` don't require any special context, just + * some initialized context) */ + secp256k1_context* ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); + if (!fill_random(randomize, sizeof(randomize))) { + printf("Failed to generate randomness\n"); + return 1; + } + /* Randomizing the context is recommended to protect against side-channel + * leakage See `secp256k1_context_randomize` in secp256k1.h for more + * information about it. This should never fail. */ + return_val = secp256k1_context_randomize(ctx, randomize); + assert(return_val); + + /*** Key Generation ***/ + + /* If the secret key is zero or out of range (bigger than secp256k1's + * order), we try to sample a new key. Note that the probability of this + * happening is negligible. */ + while (1) { + if (!fill_random(seckey1, sizeof(seckey1)) || !fill_random(seckey2, sizeof(seckey2))) { + printf("Failed to generate randomness\n"); + return 1; + } + if (secp256k1_ec_seckey_verify(ctx, seckey1) && secp256k1_ec_seckey_verify(ctx, seckey2)) { + break; + } + } + + /* Public key creation using a valid context with a verified secret key should never fail */ + return_val = secp256k1_ec_pubkey_create(ctx, &pubkey1, seckey1); + assert(return_val); + return_val = secp256k1_ec_pubkey_create(ctx, &pubkey2, seckey2); + assert(return_val); + + /* Serialize pubkey1 in a compressed form (33 bytes), should always return 1 */ + len = sizeof(compressed_pubkey1); + return_val = secp256k1_ec_pubkey_serialize(ctx, compressed_pubkey1, &len, &pubkey1, SECP256K1_EC_COMPRESSED); + assert(return_val); + /* Should be the same size as the size of the output, because we passed a 33 byte array. */ + assert(len == sizeof(compressed_pubkey1)); + + /* Serialize pubkey2 in a compressed form (33 bytes) */ + len = sizeof(compressed_pubkey2); + return_val = secp256k1_ec_pubkey_serialize(ctx, compressed_pubkey2, &len, &pubkey2, SECP256K1_EC_COMPRESSED); + assert(return_val); + /* Should be the same size as the size of the output, because we passed a 33 byte array. */ + assert(len == sizeof(compressed_pubkey2)); + + /*** Creating the shared secret ***/ + + /* Perform ECDH with seckey1 and pubkey2. Should never fail with a verified + * seckey and valid pubkey */ + return_val = secp256k1_ecdh(ctx, shared_secret1, &pubkey2, seckey1, NULL, NULL); + assert(return_val); + + /* Perform ECDH with seckey2 and pubkey1. Should never fail with a verified + * seckey and valid pubkey */ + return_val = secp256k1_ecdh(ctx, shared_secret2, &pubkey1, seckey2, NULL, NULL); + assert(return_val); + + /* Both parties should end up with the same shared secret */ + return_val = memcmp(shared_secret1, shared_secret2, sizeof(shared_secret1)); + assert(return_val == 0); + + printf("Secret Key1: "); + print_hex(seckey1, sizeof(seckey1)); + printf("Compressed Pubkey1: "); + print_hex(compressed_pubkey1, sizeof(compressed_pubkey1)); + printf("\nSecret Key2: "); + print_hex(seckey2, sizeof(seckey2)); + printf("Compressed Pubkey2: "); + print_hex(compressed_pubkey2, sizeof(compressed_pubkey2)); + printf("\nShared Secret: "); + print_hex(shared_secret1, sizeof(shared_secret1)); + + /* This will clear everything from the context and free the memory */ + secp256k1_context_destroy(ctx); + + /* It's best practice to try to clear secrets from memory after using them. + * This is done because some bugs can allow an attacker to leak memory, for + * example through "out of bounds" array access (see Heartbleed), Or the OS + * swapping them to disk. Hence, we overwrite the secret key buffer with zeros. + * + * TODO: Prevent these writes from being optimized out, as any good compiler + * will remove any writes that aren't used. */ + memset(seckey1, 0, sizeof(seckey1)); + memset(seckey2, 0, sizeof(seckey2)); + memset(shared_secret1, 0, sizeof(shared_secret1)); + memset(shared_secret2, 0, sizeof(shared_secret2)); + + return 0; +} diff --git a/src/secp256k1/examples/ecdsa.c b/src/secp256k1/examples/ecdsa.c new file mode 100644 index 0000000000..434c856ba0 --- /dev/null +++ b/src/secp256k1/examples/ecdsa.c @@ -0,0 +1,137 @@ +/************************************************************************* + * Written in 2020-2022 by Elichai Turkel * + * To the extent possible under law, the author(s) have dedicated all * + * copyright and related and neighboring rights to the software in this * + * file to the public domain worldwide. This software is distributed * + * without any warranty. For the CC0 Public Domain Dedication, see * + * EXAMPLES_COPYING or https://creativecommons.org/publicdomain/zero/1.0 * + *************************************************************************/ + +#include <stdio.h> +#include <assert.h> +#include <string.h> + +#include <secp256k1.h> + +#include "random.h" + + + +int main(void) { + /* Instead of signing the message directly, we must sign a 32-byte hash. + * Here the message is "Hello, world!" and the hash function was SHA-256. + * An actual implementation should just call SHA-256, but this example + * hardcodes the output to avoid depending on an additional library. + * See https://bitcoin.stackexchange.com/questions/81115/if-someone-wanted-to-pretend-to-be-satoshi-by-posting-a-fake-signature-to-defrau/81116#81116 */ + unsigned char msg_hash[32] = { + 0x31, 0x5F, 0x5B, 0xDB, 0x76, 0xD0, 0x78, 0xC4, + 0x3B, 0x8A, 0xC0, 0x06, 0x4E, 0x4A, 0x01, 0x64, + 0x61, 0x2B, 0x1F, 0xCE, 0x77, 0xC8, 0x69, 0x34, + 0x5B, 0xFC, 0x94, 0xC7, 0x58, 0x94, 0xED, 0xD3, + }; + unsigned char seckey[32]; + unsigned char randomize[32]; + unsigned char compressed_pubkey[33]; + unsigned char serialized_signature[64]; + size_t len; + int is_signature_valid; + int return_val; + secp256k1_pubkey pubkey; + secp256k1_ecdsa_signature sig; + /* The specification in secp256k1.h states that `secp256k1_ec_pubkey_create` needs + * a context object initialized for signing and `secp256k1_ecdsa_verify` needs + * a context initialized for verification, which is why we create a context + * for both signing and verification with the SECP256K1_CONTEXT_SIGN and + * SECP256K1_CONTEXT_VERIFY flags. */ + secp256k1_context* ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); + if (!fill_random(randomize, sizeof(randomize))) { + printf("Failed to generate randomness\n"); + return 1; + } + /* Randomizing the context is recommended to protect against side-channel + * leakage See `secp256k1_context_randomize` in secp256k1.h for more + * information about it. This should never fail. */ + return_val = secp256k1_context_randomize(ctx, randomize); + assert(return_val); + + /*** Key Generation ***/ + + /* If the secret key is zero or out of range (bigger than secp256k1's + * order), we try to sample a new key. Note that the probability of this + * happening is negligible. */ + while (1) { + if (!fill_random(seckey, sizeof(seckey))) { + printf("Failed to generate randomness\n"); + return 1; + } + if (secp256k1_ec_seckey_verify(ctx, seckey)) { + break; + } + } + + /* Public key creation using a valid context with a verified secret key should never fail */ + return_val = secp256k1_ec_pubkey_create(ctx, &pubkey, seckey); + assert(return_val); + + /* Serialize the pubkey in a compressed form(33 bytes). Should always return 1. */ + len = sizeof(compressed_pubkey); + return_val = secp256k1_ec_pubkey_serialize(ctx, compressed_pubkey, &len, &pubkey, SECP256K1_EC_COMPRESSED); + assert(return_val); + /* Should be the same size as the size of the output, because we passed a 33 byte array. */ + assert(len == sizeof(compressed_pubkey)); + + /*** Signing ***/ + + /* Generate an ECDSA signature `noncefp` and `ndata` allows you to pass a + * custom nonce function, passing `NULL` will use the RFC-6979 safe default. + * Signing with a valid context, verified secret key + * and the default nonce function should never fail. */ + return_val = secp256k1_ecdsa_sign(ctx, &sig, msg_hash, seckey, NULL, NULL); + assert(return_val); + + /* Serialize the signature in a compact form. Should always return 1 + * according to the documentation in secp256k1.h. */ + return_val = secp256k1_ecdsa_signature_serialize_compact(ctx, serialized_signature, &sig); + assert(return_val); + + + /*** Verification ***/ + + /* Deserialize the signature. This will return 0 if the signature can't be parsed correctly. */ + if (!secp256k1_ecdsa_signature_parse_compact(ctx, &sig, serialized_signature)) { + printf("Failed parsing the signature\n"); + return 1; + } + + /* Deserialize the public key. This will return 0 if the public key can't be parsed correctly. */ + if (!secp256k1_ec_pubkey_parse(ctx, &pubkey, compressed_pubkey, sizeof(compressed_pubkey))) { + printf("Failed parsing the public key\n"); + return 1; + } + + /* Verify a signature. This will return 1 if it's valid and 0 if it's not. */ + is_signature_valid = secp256k1_ecdsa_verify(ctx, &sig, msg_hash, &pubkey); + + printf("Is the signature valid? %s\n", is_signature_valid ? "true" : "false"); + printf("Secret Key: "); + print_hex(seckey, sizeof(seckey)); + printf("Public Key: "); + print_hex(compressed_pubkey, sizeof(compressed_pubkey)); + printf("Signature: "); + print_hex(serialized_signature, sizeof(serialized_signature)); + + + /* This will clear everything from the context and free the memory */ + secp256k1_context_destroy(ctx); + + /* It's best practice to try to clear secrets from memory after using them. + * This is done because some bugs can allow an attacker to leak memory, for + * example through "out of bounds" array access (see Heartbleed), Or the OS + * swapping them to disk. Hence, we overwrite the secret key buffer with zeros. + * + * TODO: Prevent these writes from being optimized out, as any good compiler + * will remove any writes that aren't used. */ + memset(seckey, 0, sizeof(seckey)); + + return 0; +} diff --git a/src/secp256k1/examples/random.h b/src/secp256k1/examples/random.h new file mode 100644 index 0000000000..439226f09f --- /dev/null +++ b/src/secp256k1/examples/random.h @@ -0,0 +1,73 @@ +/************************************************************************* + * Copyright (c) 2020-2021 Elichai Turkel * + * Distributed under the CC0 software license, see the accompanying file * + * EXAMPLES_COPYING or https://creativecommons.org/publicdomain/zero/1.0 * + *************************************************************************/ + +/* + * This file is an attempt at collecting best practice methods for obtaining randomness with different operating systems. + * It may be out-of-date. Consult the documentation of the operating system before considering to use the methods below. + * + * Platform randomness sources: + * Linux -> `getrandom(2)`(`sys/random.h`), if not available `/dev/urandom` should be used. http://man7.org/linux/man-pages/man2/getrandom.2.html, https://linux.die.net/man/4/urandom + * macOS -> `getentropy(2)`(`sys/random.h`), if not available `/dev/urandom` should be used. https://www.unix.com/man-page/mojave/2/getentropy, https://opensource.apple.com/source/xnu/xnu-517.12.7/bsd/man/man4/random.4.auto.html + * FreeBSD -> `getrandom(2)`(`sys/random.h`), if not available `kern.arandom` should be used. https://www.freebsd.org/cgi/man.cgi?query=getrandom, https://www.freebsd.org/cgi/man.cgi?query=random&sektion=4 + * OpenBSD -> `getentropy(2)`(`unistd.h`), if not available `/dev/urandom` should be used. https://man.openbsd.org/getentropy, https://man.openbsd.org/urandom + * Windows -> `BCryptGenRandom`(`bcrypt.h`). https://docs.microsoft.com/en-us/windows/win32/api/bcrypt/nf-bcrypt-bcryptgenrandom + */ + +#if defined(_WIN32) +#include <windows.h> +#include <ntstatus.h> +#include <bcrypt.h> +#elif defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) +#include <sys/random.h> +#elif defined(__OpenBSD__) +#include <unistd.h> +#else +#error "Couldn't identify the OS" +#endif + +#include <stddef.h> +#include <limits.h> +#include <stdio.h> + + +/* Returns 1 on success, and 0 on failure. */ +static int fill_random(unsigned char* data, size_t size) { +#if defined(_WIN32) + NTSTATUS res = BCryptGenRandom(NULL, data, size, BCRYPT_USE_SYSTEM_PREFERRED_RNG); + if (res != STATUS_SUCCESS || size > ULONG_MAX) { + return 0; + } else { + return 1; + } +#elif defined(__linux__) || defined(__FreeBSD__) + /* If `getrandom(2)` is not available you should fallback to /dev/urandom */ + ssize_t res = getrandom(data, size, 0); + if (res < 0 || (size_t)res != size ) { + return 0; + } else { + return 1; + } +#elif defined(__APPLE__) || defined(__OpenBSD__) + /* If `getentropy(2)` is not available you should fallback to either + * `SecRandomCopyBytes` or /dev/urandom */ + int res = getentropy(data, size); + if (res == 0) { + return 1; + } else { + return 0; + } +#endif + return 0; +} + +static void print_hex(unsigned char* data, size_t size) { + size_t i; + printf("0x"); + for (i = 0; i < size; i++) { + printf("%02x", data[i]); + } + printf("\n"); +} diff --git a/src/secp256k1/examples/schnorr.c b/src/secp256k1/examples/schnorr.c new file mode 100644 index 0000000000..82eb07d5d7 --- /dev/null +++ b/src/secp256k1/examples/schnorr.c @@ -0,0 +1,152 @@ +/************************************************************************* + * Written in 2020-2022 by Elichai Turkel * + * To the extent possible under law, the author(s) have dedicated all * + * copyright and related and neighboring rights to the software in this * + * file to the public domain worldwide. This software is distributed * + * without any warranty. For the CC0 Public Domain Dedication, see * + * EXAMPLES_COPYING or https://creativecommons.org/publicdomain/zero/1.0 * + *************************************************************************/ + +#include <stdio.h> +#include <assert.h> +#include <string.h> + +#include <secp256k1.h> +#include <secp256k1_extrakeys.h> +#include <secp256k1_schnorrsig.h> + +#include "random.h" + +int main(void) { + unsigned char msg[12] = "Hello World!"; + unsigned char msg_hash[32]; + unsigned char tag[17] = "my_fancy_protocol"; + unsigned char seckey[32]; + unsigned char randomize[32]; + unsigned char auxiliary_rand[32]; + unsigned char serialized_pubkey[32]; + unsigned char signature[64]; + int is_signature_valid; + int return_val; + secp256k1_xonly_pubkey pubkey; + secp256k1_keypair keypair; + /* The specification in secp256k1_extrakeys.h states that `secp256k1_keypair_create` + * needs a context object initialized for signing. And in secp256k1_schnorrsig.h + * they state that `secp256k1_schnorrsig_verify` needs a context initialized for + * verification, which is why we create a context for both signing and verification + * with the SECP256K1_CONTEXT_SIGN and SECP256K1_CONTEXT_VERIFY flags. */ + secp256k1_context* ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); + if (!fill_random(randomize, sizeof(randomize))) { + printf("Failed to generate randomness\n"); + return 1; + } + /* Randomizing the context is recommended to protect against side-channel + * leakage See `secp256k1_context_randomize` in secp256k1.h for more + * information about it. This should never fail. */ + return_val = secp256k1_context_randomize(ctx, randomize); + assert(return_val); + + /*** Key Generation ***/ + + /* If the secret key is zero or out of range (bigger than secp256k1's + * order), we try to sample a new key. Note that the probability of this + * happening is negligible. */ + while (1) { + if (!fill_random(seckey, sizeof(seckey))) { + printf("Failed to generate randomness\n"); + return 1; + } + /* Try to create a keypair with a valid context, it should only fail if + * the secret key is zero or out of range. */ + if (secp256k1_keypair_create(ctx, &keypair, seckey)) { + break; + } + } + + /* Extract the X-only public key from the keypair. We pass NULL for + * `pk_parity` as the parity isn't needed for signing or verification. + * `secp256k1_keypair_xonly_pub` supports returning the parity for + * other use cases such as tests or verifying Taproot tweaks. + * This should never fail with a valid context and public key. */ + return_val = secp256k1_keypair_xonly_pub(ctx, &pubkey, NULL, &keypair); + assert(return_val); + + /* Serialize the public key. Should always return 1 for a valid public key. */ + return_val = secp256k1_xonly_pubkey_serialize(ctx, serialized_pubkey, &pubkey); + assert(return_val); + + /*** Signing ***/ + + /* Instead of signing (possibly very long) messages directly, we sign a + * 32-byte hash of the message in this example. + * + * We use secp256k1_tagged_sha256 to create this hash. This function expects + * a context-specific "tag", which restricts the context in which the signed + * messages should be considered valid. For example, if protocol A mandates + * to use the tag "my_fancy_protocol" and protocol B mandates to use the tag + * "my_boring_protocol", then signed messages from protocol A will never be + * valid in protocol B (and vice versa), even if keys are reused across + * protocols. This implements "domain separation", which is considered good + * practice. It avoids attacks in which users are tricked into signing a + * message that has intended consequences in the intended context (e.g., + * protocol A) but would have unintended consequences if it were valid in + * some other context (e.g., protocol B). */ + return_val = secp256k1_tagged_sha256(ctx, msg_hash, tag, sizeof(tag), msg, sizeof(msg)); + assert(return_val); + + /* Generate 32 bytes of randomness to use with BIP-340 schnorr signing. */ + if (!fill_random(auxiliary_rand, sizeof(auxiliary_rand))) { + printf("Failed to generate randomness\n"); + return 1; + } + + /* Generate a Schnorr signature. + * + * We use the secp256k1_schnorrsig_sign32 function that provides a simple + * interface for signing 32-byte messages (which in our case is a hash of + * the actual message). BIP-340 recommends passing 32 bytes of randomness + * to the signing function to improve security against side-channel attacks. + * Signing with a valid context, a 32-byte message, a verified keypair, and + * any 32 bytes of auxiliary random data should never fail. */ + return_val = secp256k1_schnorrsig_sign32(ctx, signature, msg_hash, &keypair, auxiliary_rand); + assert(return_val); + + /*** Verification ***/ + + /* Deserialize the public key. This will return 0 if the public key can't + * be parsed correctly */ + if (!secp256k1_xonly_pubkey_parse(ctx, &pubkey, serialized_pubkey)) { + printf("Failed parsing the public key\n"); + return 1; + } + + /* Compute the tagged hash on the received messages using the same tag as the signer. */ + return_val = secp256k1_tagged_sha256(ctx, msg_hash, tag, sizeof(tag), msg, sizeof(msg)); + assert(return_val); + + /* Verify a signature. This will return 1 if it's valid and 0 if it's not. */ + is_signature_valid = secp256k1_schnorrsig_verify(ctx, signature, msg_hash, 32, &pubkey); + + + printf("Is the signature valid? %s\n", is_signature_valid ? "true" : "false"); + printf("Secret Key: "); + print_hex(seckey, sizeof(seckey)); + printf("Public Key: "); + print_hex(serialized_pubkey, sizeof(serialized_pubkey)); + printf("Signature: "); + print_hex(signature, sizeof(signature)); + + /* This will clear everything from the context and free the memory */ + secp256k1_context_destroy(ctx); + + /* It's best practice to try to clear secrets from memory after using them. + * This is done because some bugs can allow an attacker to leak memory, for + * example through "out of bounds" array access (see Heartbleed), Or the OS + * swapping them to disk. Hence, we overwrite the secret key buffer with zeros. + * + * TODO: Prevent these writes from being optimized out, as any good compiler + * will remove any writes that aren't used. */ + memset(seckey, 0, sizeof(seckey)); + + return 0; +} diff --git a/src/secp256k1/include/secp256k1.h b/src/secp256k1/include/secp256k1.h index 57114b8f26..86ab7e556d 100644 --- a/src/secp256k1/include/secp256k1.h +++ b/src/secp256k1/include/secp256k1.h @@ -169,6 +169,17 @@ typedef int (*secp256k1_nonce_function)( # define SECP256K1_ARG_NONNULL(_x) # endif +/** Attribute for marking functions, types, and variables as deprecated */ +#if !defined(SECP256K1_BUILD) && defined(__has_attribute) +# if __has_attribute(__deprecated__) +# define SECP256K1_DEPRECATED(_msg) __attribute__ ((__deprecated__(_msg))) +# else +# define SECP256K1_DEPRECATED(_msg) +# endif +#else +# define SECP256K1_DEPRECATED(_msg) +#endif + /** All flags' lower 8 bits indicate what they're for. Do not use directly. */ #define SECP256K1_FLAGS_TYPE_MASK ((1 << 8) - 1) #define SECP256K1_FLAGS_TYPE_CONTEXT (1 << 0) @@ -641,7 +652,8 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_seckey_negate( SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_negate( const secp256k1_context* ctx, unsigned char *seckey -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) + SECP256K1_DEPRECATED("Use secp256k1_ec_seckey_negate instead"); /** Negates a public key in place. * @@ -681,7 +693,8 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_tweak_add( const secp256k1_context* ctx, unsigned char *seckey, const unsigned char *tweak32 -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) + SECP256K1_DEPRECATED("Use secp256k1_ec_seckey_tweak_add instead"); /** Tweak a public key by adding tweak times the generator to it. * @@ -727,7 +740,8 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_tweak_mul( const secp256k1_context* ctx, unsigned char *seckey, const unsigned char *tweak32 -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) + SECP256K1_DEPRECATED("Use secp256k1_ec_seckey_tweak_mul instead"); /** Tweak a public key by multiplying it by a tweak value. * @@ -800,7 +814,7 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_combine( * implementations optimized for a specific tag can precompute the SHA256 state * after hashing the tag hashes. * - * Returns 0 if the arguments are invalid and 1 otherwise. + * Returns: 1 always. * Args: ctx: pointer to a context object * Out: hash32: pointer to a 32-byte array to store the resulting hash * In: tag: pointer to an array containing the tag diff --git a/src/secp256k1/include/secp256k1_extrakeys.h b/src/secp256k1/include/secp256k1_extrakeys.h index a64d561b60..09cbeaaa80 100644 --- a/src/secp256k1/include/secp256k1_extrakeys.h +++ b/src/secp256k1/include/secp256k1_extrakeys.h @@ -81,8 +81,7 @@ SECP256K1_API int secp256k1_xonly_pubkey_cmp( /** Converts a secp256k1_pubkey into a secp256k1_xonly_pubkey. * - * Returns: 1 if the public key was successfully converted - * 0 otherwise + * Returns: 1 always. * * Args: ctx: pointer to a context object. * Out: xonly_pubkey: pointer to an x-only public key object for placing the converted public key. @@ -172,7 +171,7 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_keypair_create( /** Get the secret key from a keypair. * - * Returns: 0 if the arguments are invalid. 1 otherwise. + * Returns: 1 always. * Args: ctx: pointer to a context object. * Out: seckey: pointer to a 32-byte buffer for the secret key. * In: keypair: pointer to a keypair. @@ -185,7 +184,7 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_keypair_sec( /** Get the public key from a keypair. * - * Returns: 0 if the arguments are invalid. 1 otherwise. + * Returns: 1 always. * Args: ctx: pointer to a context object. * Out: pubkey: pointer to a pubkey object. If 1 is returned, it is set to * the keypair public key. If not, it's set to an invalid value. @@ -202,7 +201,7 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_keypair_pub( * This is the same as calling secp256k1_keypair_pub and then * secp256k1_xonly_pubkey_from_pubkey. * - * Returns: 0 if the arguments are invalid. 1 otherwise. + * Returns: 1 always. * Args: ctx: pointer to a context object. * Out: pubkey: pointer to an xonly_pubkey object. If 1 is returned, it is set * to the keypair public key after converting it to an diff --git a/src/secp256k1/include/secp256k1_schnorrsig.h b/src/secp256k1/include/secp256k1_schnorrsig.h index e971ddc2aa..5fedcb07b0 100644 --- a/src/secp256k1/include/secp256k1_schnorrsig.h +++ b/src/secp256k1/include/secp256k1_schnorrsig.h @@ -116,7 +116,7 @@ typedef struct { * BIP-340 "Default Signing" for a full explanation of this * argument and for guidance if randomness is expensive. */ -SECP256K1_API int secp256k1_schnorrsig_sign( +SECP256K1_API int secp256k1_schnorrsig_sign32( const secp256k1_context* ctx, unsigned char *sig64, const unsigned char *msg32, @@ -124,6 +124,17 @@ SECP256K1_API int secp256k1_schnorrsig_sign( const unsigned char *aux_rand32 ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); +/** Same as secp256k1_schnorrsig_sign32, but DEPRECATED. Will be removed in + * future versions. */ +SECP256K1_API int secp256k1_schnorrsig_sign( + const secp256k1_context* ctx, + unsigned char *sig64, + const unsigned char *msg32, + const secp256k1_keypair *keypair, + const unsigned char *aux_rand32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) + SECP256K1_DEPRECATED("Use secp256k1_schnorrsig_sign32 instead"); + /** Create a Schnorr signature with a more flexible API. * * Same arguments as secp256k1_schnorrsig_sign except that it allows signing diff --git a/src/secp256k1/sage/group_prover.sage b/src/secp256k1/sage/group_prover.sage index b200bfeae3..9305c215d5 100644 --- a/src/secp256k1/sage/group_prover.sage +++ b/src/secp256k1/sage/group_prover.sage @@ -164,6 +164,9 @@ class constraints: def negate(self): return constraints(zero=self.nonzero, nonzero=self.zero) + def map(self, fun): + return constraints(zero={fun(k): v for k, v in self.zero.items()}, nonzero={fun(k): v for k, v in self.nonzero.items()}) + def __add__(self, other): zero = self.zero.copy() zero.update(other.zero) @@ -177,6 +180,30 @@ class constraints: def __repr__(self): return "%s" % self +def normalize_factor(p): + """Normalizes the sign of primitive polynomials (as returned by factor()) + + This function ensures that the polynomial has a positive leading coefficient. + + This is necessary because recent sage versions (starting with v9.3 or v9.4, + we don't know) are inconsistent about the placement of the minus sign in + polynomial factorizations: + ``` + sage: R.<ax,bx,ay,by,Az,Bz,Ai,Bi> = PolynomialRing(QQ,8,order='invlex') + sage: R((-2 * (bx - ax)) ^ 1).factor() + (-2) * (bx - ax) + sage: R((-2 * (bx - ax)) ^ 2).factor() + (4) * (-bx + ax)^2 + sage: R((-2 * (bx - ax)) ^ 3).factor() + (8) * (-bx + ax)^3 + ``` + """ + # Assert p is not 0 and that its non-zero coeffients are coprime. + # (We could just work with the primitive part p/p.content() but we want to be + # aware if factor() does not return a primitive part in future sage versions.) + assert p.content() == 1 + # Ensure that the first non-zero coefficient is positive. + return p if p.lc() > 0 else -p def conflicts(R, con): """Check whether any of the passed non-zero assumptions is implied by the zero assumptions""" @@ -204,10 +231,10 @@ def get_nonzero_set(R, assume): nonzero = set() for nz in map(numerator, assume.nonzero): for (f,n) in nz.factor(): - nonzero.add(f) + nonzero.add(normalize_factor(f)) rnz = zero.reduce(nz) for (f,n) in rnz.factor(): - nonzero.add(f) + nonzero.add(normalize_factor(f)) return nonzero @@ -222,27 +249,27 @@ def prove_nonzero(R, exprs, assume): return (False, [exprs[expr]]) allexprs = reduce(lambda a,b: numerator(a)*numerator(b), exprs, 1) for (f, n) in allexprs.factor(): - if f not in nonzero: + if normalize_factor(f) not in nonzero: ok = False if ok: return (True, None) ok = True - for (f, n) in zero.reduce(numerator(allexprs)).factor(): - if f not in nonzero: + for (f, n) in zero.reduce(allexprs).factor(): + if normalize_factor(f) not in nonzero: ok = False if ok: return (True, None) ok = True for expr in exprs: for (f,n) in numerator(expr).factor(): - if f not in nonzero: + if normalize_factor(f) not in nonzero: ok = False if ok: return (True, None) ok = True for expr in exprs: for (f,n) in zero.reduce(numerator(expr)).factor(): - if f not in nonzero: + if normalize_factor(f) not in nonzero: expl.add(exprs[expr]) if expl: return (False, list(expl)) @@ -254,7 +281,7 @@ def prove_zero(R, exprs, assume): """Check whether all of the passed expressions are provably zero, given assumptions""" r, e = prove_nonzero(R, dict(map(lambda x: (fastfrac(R, x.bot, 1), exprs[x]), exprs)), assume) if not r: - return (False, map(lambda x: "Possibly zero denominator: %s" % x, e)) + return (False, list(map(lambda x: "Possibly zero denominator: %s" % x, e))) zero = R.ideal(list(map(numerator, assume.zero))) nonzero = prod(x for x in assume.nonzero) expl = [] @@ -279,8 +306,8 @@ def describe_extra(R, assume, assumeExtra): if base not in zero: add = [] for (f, n) in numerator(base).factor(): - if f not in nonzero: - add += ["%s" % f] + if normalize_factor(f) not in nonzero: + add += ["%s" % normalize_factor(f)] if add: ret.add((" * ".join(add)) + " = 0 [%s]" % assumeExtra.zero[base]) # Iterate over the extra nonzero expressions @@ -288,8 +315,8 @@ def describe_extra(R, assume, assumeExtra): nzr = zeroextra.reduce(numerator(nz)) if nzr not in zeroextra: for (f,n) in nzr.factor(): - if zeroextra.reduce(f) not in nonzero: - ret.add("%s != 0" % zeroextra.reduce(f)) + if normalize_factor(zeroextra.reduce(f)) not in nonzero: + ret.add("%s != 0" % normalize_factor(zeroextra.reduce(f))) return ", ".join(x for x in ret) @@ -299,22 +326,21 @@ def check_symbolic(R, assumeLaw, assumeAssert, assumeBranch, require): if conflicts(R, assume): # This formula does not apply - return None + return (True, None) describe = describe_extra(R, assumeLaw + assumeBranch, assumeAssert) + if describe != "": + describe = " (assuming " + describe + ")" ok, msg = prove_zero(R, require.zero, assume) if not ok: - return "FAIL, %s fails (assuming %s)" % (str(msg), describe) + return (False, "FAIL, %s fails%s" % (str(msg), describe)) res, expl = prove_nonzero(R, require.nonzero, assume) if not res: - return "FAIL, %s fails (assuming %s)" % (str(expl), describe) + return (False, "FAIL, %s fails%s" % (str(expl), describe)) - if describe != "": - return "OK (assuming %s)" % describe - else: - return "OK" + return (True, "OK%s" % describe) def concrete_verify(c): diff --git a/src/secp256k1/sage/prove_group_implementations.sage b/src/secp256k1/sage/prove_group_implementations.sage index a97e732f7f..96ce33506a 100644 --- a/src/secp256k1/sage/prove_group_implementations.sage +++ b/src/secp256k1/sage/prove_group_implementations.sage @@ -8,25 +8,20 @@ load("weierstrass_prover.sage") def formula_secp256k1_gej_double_var(a): """libsecp256k1's secp256k1_gej_double_var, used by various addition functions""" rz = a.Z * a.Y - rz = rz * 2 - t1 = a.X^2 - t1 = t1 * 3 - t2 = t1^2 - t3 = a.Y^2 - t3 = t3 * 2 - t4 = t3^2 - t4 = t4 * 2 - t3 = t3 * a.X - rx = t3 - rx = rx * 4 - rx = -rx - rx = rx + t2 - t2 = -t2 - t3 = t3 * 6 - t3 = t3 + t2 - ry = t1 * t3 - t2 = -t4 - ry = ry + t2 + s = a.Y^2 + l = a.X^2 + l = l * 3 + l = l / 2 + t = -s + t = t * a.X + rx = l^2 + rx = rx + t + rx = rx + t + s = s^2 + t = t + rx + ry = t * l + ry = ry + s + ry = -ry return jacobianpoint(rx, ry, rz) def formula_secp256k1_gej_add_var(branch, a, b): @@ -197,7 +192,8 @@ def formula_secp256k1_gej_add_ge(branch, a, b): rr_alt = rr m_alt = m n = m_alt^2 - q = n * t + q = -t + q = q * n n = n^2 if degenerate: n = m @@ -210,8 +206,6 @@ def formula_secp256k1_gej_add_ge(branch, a, b): zeroes.update({rz : 'r.z=0'}) else: nonzeroes.update({rz : 'r.z!=0'}) - rz = rz * 2 - q = -q t = t + q rx = t t = t * 2 @@ -219,8 +213,7 @@ def formula_secp256k1_gej_add_ge(branch, a, b): t = t * rr_alt t = t + n ry = -t - rx = rx * 4 - ry = ry * 4 + ry = ry / 2 if a_infinity: rx = b.X ry = b.Y @@ -292,15 +285,18 @@ def formula_secp256k1_gej_add_ge_old(branch, a, b): return (constraints(zero={b.Z - 1 : 'b.z=1', b.Infinity : 'b_finite'}), constraints(zero=zero, nonzero=nonzero), jacobianpoint(rx, ry, rz)) if __name__ == "__main__": - check_symbolic_jacobian_weierstrass("secp256k1_gej_add_var", 0, 7, 5, formula_secp256k1_gej_add_var) - check_symbolic_jacobian_weierstrass("secp256k1_gej_add_ge_var", 0, 7, 5, formula_secp256k1_gej_add_ge_var) - check_symbolic_jacobian_weierstrass("secp256k1_gej_add_zinv_var", 0, 7, 5, formula_secp256k1_gej_add_zinv_var) - check_symbolic_jacobian_weierstrass("secp256k1_gej_add_ge", 0, 7, 16, formula_secp256k1_gej_add_ge) - check_symbolic_jacobian_weierstrass("secp256k1_gej_add_ge_old [should fail]", 0, 7, 4, formula_secp256k1_gej_add_ge_old) + success = True + success = success & check_symbolic_jacobian_weierstrass("secp256k1_gej_add_var", 0, 7, 5, formula_secp256k1_gej_add_var) + success = success & check_symbolic_jacobian_weierstrass("secp256k1_gej_add_ge_var", 0, 7, 5, formula_secp256k1_gej_add_ge_var) + success = success & check_symbolic_jacobian_weierstrass("secp256k1_gej_add_zinv_var", 0, 7, 5, formula_secp256k1_gej_add_zinv_var) + success = success & check_symbolic_jacobian_weierstrass("secp256k1_gej_add_ge", 0, 7, 16, formula_secp256k1_gej_add_ge) + success = success & (not check_symbolic_jacobian_weierstrass("secp256k1_gej_add_ge_old [should fail]", 0, 7, 4, formula_secp256k1_gej_add_ge_old)) if len(sys.argv) >= 2 and sys.argv[1] == "--exhaustive": - check_exhaustive_jacobian_weierstrass("secp256k1_gej_add_var", 0, 7, 5, formula_secp256k1_gej_add_var, 43) - check_exhaustive_jacobian_weierstrass("secp256k1_gej_add_ge_var", 0, 7, 5, formula_secp256k1_gej_add_ge_var, 43) - check_exhaustive_jacobian_weierstrass("secp256k1_gej_add_zinv_var", 0, 7, 5, formula_secp256k1_gej_add_zinv_var, 43) - check_exhaustive_jacobian_weierstrass("secp256k1_gej_add_ge", 0, 7, 16, formula_secp256k1_gej_add_ge, 43) - check_exhaustive_jacobian_weierstrass("secp256k1_gej_add_ge_old [should fail]", 0, 7, 4, formula_secp256k1_gej_add_ge_old, 43) + success = success & check_exhaustive_jacobian_weierstrass("secp256k1_gej_add_var", 0, 7, 5, formula_secp256k1_gej_add_var, 43) + success = success & check_exhaustive_jacobian_weierstrass("secp256k1_gej_add_ge_var", 0, 7, 5, formula_secp256k1_gej_add_ge_var, 43) + success = success & check_exhaustive_jacobian_weierstrass("secp256k1_gej_add_zinv_var", 0, 7, 5, formula_secp256k1_gej_add_zinv_var, 43) + success = success & check_exhaustive_jacobian_weierstrass("secp256k1_gej_add_ge", 0, 7, 16, formula_secp256k1_gej_add_ge, 43) + success = success & (not check_exhaustive_jacobian_weierstrass("secp256k1_gej_add_ge_old [should fail]", 0, 7, 4, formula_secp256k1_gej_add_ge_old, 43)) + + sys.exit(int(not success)) diff --git a/src/secp256k1/sage/weierstrass_prover.sage b/src/secp256k1/sage/weierstrass_prover.sage index b770c6dafe..be9cfd4c76 100644 --- a/src/secp256k1/sage/weierstrass_prover.sage +++ b/src/secp256k1/sage/weierstrass_prover.sage @@ -184,6 +184,7 @@ def check_exhaustive_jacobian_weierstrass(name, A, B, branches, formula, p): if r: points.append(point) + ret = True for za in range(1, p): for zb in range(1, p): for pa in points: @@ -211,8 +212,11 @@ def check_exhaustive_jacobian_weierstrass(name, A, B, branches, formula, p): match = True r, e = concrete_verify(require) if not r: + ret = False print(" failure in branch %i for (%s,%s,%s,%s) + (%s,%s,%s,%s) = (%s,%s,%s,%s): %s" % (branch, pA.X, pA.Y, pA.Z, pA.Infinity, pB.X, pB.Y, pB.Z, pB.Infinity, pC.X, pC.Y, pC.Z, pC.Infinity, e)) + print() + return ret def check_symbolic_function(R, assumeAssert, assumeBranch, f, A, B, pa, pb, pA, pB, pC): @@ -244,15 +248,21 @@ def check_symbolic_jacobian_weierstrass(name, A, B, branches, formula): print("Formula " + name + ":") count = 0 + ret = True for branch in range(branches): assumeFormula, assumeBranch, pC = formula(branch, pA, pB) + assumeBranch = assumeBranch.map(lift) + assumeFormula = assumeFormula.map(lift) pC.X = lift(pC.X) pC.Y = lift(pC.Y) pC.Z = lift(pC.Z) pC.Infinity = lift(pC.Infinity) for key in laws_jacobian_weierstrass: - res[key].append((check_symbolic_function(R, assumeFormula, assumeBranch, laws_jacobian_weierstrass[key], A, B, pa, pb, pA, pB, pC), branch)) + success, msg = check_symbolic_function(R, assumeFormula, assumeBranch, laws_jacobian_weierstrass[key], A, B, pa, pb, pA, pB, pC) + if not success: + ret = False + res[key].append((msg, branch)) for key in res: print(" %s:" % key) @@ -262,3 +272,4 @@ def check_symbolic_jacobian_weierstrass(name, A, B, branches, formula): print(" branch %i: %s" % (x[1], x[0])) print() + return ret diff --git a/src/secp256k1/src/bench_internal.c b/src/secp256k1/src/bench_internal.c index aed8216127..3c145f306c 100644 --- a/src/secp256k1/src/bench_internal.c +++ b/src/secp256k1/src/bench_internal.c @@ -140,6 +140,15 @@ void bench_scalar_inverse_var(void* arg, int iters) { CHECK(j <= iters); } +void bench_field_half(void* arg, int iters) { + int i; + bench_inv *data = (bench_inv*)arg; + + for (i = 0; i < iters; i++) { + secp256k1_fe_half(&data->fe[0]); + } +} + void bench_field_normalize(void* arg, int iters) { int i; bench_inv *data = (bench_inv*)arg; @@ -354,6 +363,7 @@ int main(int argc, char **argv) { if (d || have_flag(argc, argv, "scalar") || have_flag(argc, argv, "inverse")) run_benchmark("scalar_inverse", bench_scalar_inverse, bench_setup, NULL, &data, 10, iters); if (d || have_flag(argc, argv, "scalar") || have_flag(argc, argv, "inverse")) run_benchmark("scalar_inverse_var", bench_scalar_inverse_var, bench_setup, NULL, &data, 10, iters); + if (d || have_flag(argc, argv, "field") || have_flag(argc, argv, "half")) run_benchmark("field_half", bench_field_half, bench_setup, NULL, &data, 10, iters*100); if (d || have_flag(argc, argv, "field") || have_flag(argc, argv, "normalize")) run_benchmark("field_normalize", bench_field_normalize, bench_setup, NULL, &data, 10, iters*100); if (d || have_flag(argc, argv, "field") || have_flag(argc, argv, "normalize")) run_benchmark("field_normalize_weak", bench_field_normalize_weak, bench_setup, NULL, &data, 10, iters*100); if (d || have_flag(argc, argv, "field") || have_flag(argc, argv, "sqr")) run_benchmark("field_sqr", bench_field_sqr, bench_setup, NULL, &data, 10, iters*10); diff --git a/src/secp256k1/src/ecmult_compute_table.h b/src/secp256k1/src/ecmult_compute_table.h new file mode 100644 index 0000000000..665f87ff3d --- /dev/null +++ b/src/secp256k1/src/ecmult_compute_table.h @@ -0,0 +1,16 @@ +/***************************************************************************************************** + * Copyright (c) 2013, 2014, 2017, 2021 Pieter Wuille, Andrew Poelstra, Jonas Nick, Russell O'Connor * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or https://www.opensource.org/licenses/mit-license.php. * + *****************************************************************************************************/ + +#ifndef SECP256K1_ECMULT_COMPUTE_TABLE_H +#define SECP256K1_ECMULT_COMPUTE_TABLE_H + +/* Construct table of all odd multiples of gen in range 1..(2**(window_g-1)-1). */ +static void secp256k1_ecmult_compute_table(secp256k1_ge_storage* table, int window_g, const secp256k1_gej* gen); + +/* Like secp256k1_ecmult_compute_table, but one for both gen and gen*2^128. */ +static void secp256k1_ecmult_compute_two_tables(secp256k1_ge_storage* table, secp256k1_ge_storage* table_128, int window_g, const secp256k1_ge* gen); + +#endif /* SECP256K1_ECMULT_COMPUTE_TABLE_H */ diff --git a/src/secp256k1/src/ecmult_compute_table_impl.h b/src/secp256k1/src/ecmult_compute_table_impl.h new file mode 100644 index 0000000000..69d59ce595 --- /dev/null +++ b/src/secp256k1/src/ecmult_compute_table_impl.h @@ -0,0 +1,49 @@ +/***************************************************************************************************** + * Copyright (c) 2013, 2014, 2017, 2021 Pieter Wuille, Andrew Poelstra, Jonas Nick, Russell O'Connor * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or https://www.opensource.org/licenses/mit-license.php. * + *****************************************************************************************************/ + +#ifndef SECP256K1_ECMULT_COMPUTE_TABLE_IMPL_H +#define SECP256K1_ECMULT_COMPUTE_TABLE_IMPL_H + +#include "ecmult_compute_table.h" +#include "group_impl.h" +#include "field_impl.h" +#include "ecmult.h" +#include "util.h" + +static void secp256k1_ecmult_compute_table(secp256k1_ge_storage* table, int window_g, const secp256k1_gej* gen) { + secp256k1_gej gj; + secp256k1_ge ge, dgen; + int j; + + gj = *gen; + secp256k1_ge_set_gej_var(&ge, &gj); + secp256k1_ge_to_storage(&table[0], &ge); + + secp256k1_gej_double_var(&gj, gen, NULL); + secp256k1_ge_set_gej_var(&dgen, &gj); + + for (j = 1; j < ECMULT_TABLE_SIZE(window_g); ++j) { + secp256k1_gej_set_ge(&gj, &ge); + secp256k1_gej_add_ge_var(&gj, &gj, &dgen, NULL); + secp256k1_ge_set_gej_var(&ge, &gj); + secp256k1_ge_to_storage(&table[j], &ge); + } +} + +/* Like secp256k1_ecmult_compute_table, but one for both gen and gen*2^128. */ +static void secp256k1_ecmult_compute_two_tables(secp256k1_ge_storage* table, secp256k1_ge_storage* table_128, int window_g, const secp256k1_ge* gen) { + secp256k1_gej gj; + int i; + + secp256k1_gej_set_ge(&gj, gen); + secp256k1_ecmult_compute_table(table, window_g, &gj); + for (i = 0; i < 128; ++i) { + secp256k1_gej_double_var(&gj, &gj, NULL); + } + secp256k1_ecmult_compute_table(table_128, window_g, &gj); +} + +#endif /* SECP256K1_ECMULT_COMPUTE_TABLE_IMPL_H */ diff --git a/src/secp256k1/src/ecmult_const_impl.h b/src/secp256k1/src/ecmult_const_impl.h index 30b151ff9a..12dbcc6c5b 100644 --- a/src/secp256k1/src/ecmult_const_impl.h +++ b/src/secp256k1/src/ecmult_const_impl.h @@ -12,6 +12,19 @@ #include "ecmult_const.h" #include "ecmult_impl.h" +/** Fill a table 'pre' with precomputed odd multiples of a. + * + * The resulting point set is brought to a single constant Z denominator, stores the X and Y + * coordinates as ge_storage points in pre, and stores the global Z in globalz. + * It only operates on tables sized for WINDOW_A wnaf multiples. + */ +static void secp256k1_ecmult_odd_multiples_table_globalz_windowa(secp256k1_ge *pre, secp256k1_fe *globalz, const secp256k1_gej *a) { + secp256k1_fe zr[ECMULT_TABLE_SIZE(WINDOW_A)]; + + secp256k1_ecmult_odd_multiples_table(ECMULT_TABLE_SIZE(WINDOW_A), pre, zr, globalz, a); + secp256k1_ge_table_set_globalz(ECMULT_TABLE_SIZE(WINDOW_A), pre, zr); +} + /* This is like `ECMULT_TABLE_GET_GE` but is constant time */ #define ECMULT_CONST_TABLE_GET_GE(r,pre,n,w) do { \ int m = 0; \ @@ -40,7 +53,6 @@ secp256k1_fe_cmov(&(r)->y, &neg_y, (n) != abs_n); \ } while(0) - /** Convert a number to WNAF notation. * The number becomes represented by sum(2^{wi} * wnaf[i], i=0..WNAF_SIZE(w)+1) - return_val. * It has the following guarantees: @@ -56,7 +68,7 @@ */ static int secp256k1_wnaf_const(int *wnaf, const secp256k1_scalar *scalar, int w, int size) { int global_sign; - int skew = 0; + int skew; int word = 0; /* 1 2 3 */ @@ -64,9 +76,7 @@ static int secp256k1_wnaf_const(int *wnaf, const secp256k1_scalar *scalar, int w int u; int flip; - int bit; - secp256k1_scalar s; - int not_neg_one; + secp256k1_scalar s = *scalar; VERIFY_CHECK(w > 0); VERIFY_CHECK(size > 0); @@ -74,33 +84,19 @@ static int secp256k1_wnaf_const(int *wnaf, const secp256k1_scalar *scalar, int w /* Note that we cannot handle even numbers by negating them to be odd, as is * done in other implementations, since if our scalars were specified to have * width < 256 for performance reasons, their negations would have width 256 - * and we'd lose any performance benefit. Instead, we use a technique from - * Section 4.2 of the Okeya/Tagaki paper, which is to add either 1 (for even) - * or 2 (for odd) to the number we are encoding, returning a skew value indicating + * and we'd lose any performance benefit. Instead, we use a variation of a + * technique from Section 4.2 of the Okeya/Tagaki paper, which is to add 1 to the + * number we are encoding when it is even, returning a skew value indicating * this, and having the caller compensate after doing the multiplication. * * In fact, we _do_ want to negate numbers to minimize their bit-lengths (and in * particular, to ensure that the outputs from the endomorphism-split fit into - * 128 bits). If we negate, the parity of our number flips, inverting which of - * {1, 2} we want to add to the scalar when ensuring that it's odd. Further - * complicating things, -1 interacts badly with `secp256k1_scalar_cadd_bit` and - * we need to special-case it in this logic. */ - flip = secp256k1_scalar_is_high(scalar); - /* We add 1 to even numbers, 2 to odd ones, noting that negation flips parity */ - bit = flip ^ !secp256k1_scalar_is_even(scalar); - /* We check for negative one, since adding 2 to it will cause an overflow */ - secp256k1_scalar_negate(&s, scalar); - not_neg_one = !secp256k1_scalar_is_one(&s); - s = *scalar; - secp256k1_scalar_cadd_bit(&s, bit, not_neg_one); - /* If we had negative one, flip == 1, s.d[0] == 0, bit == 1, so caller expects - * that we added two to it and flipped it. In fact for -1 these operations are - * identical. We only flipped, but since skewing is required (in the sense that - * the skew must be 1 or 2, never zero) and flipping is not, we need to change - * our flags to claim that we only skewed. */ + * 128 bits). If we negate, the parity of our number flips, affecting whether + * we want to add to the scalar to ensure that it's odd. */ + flip = secp256k1_scalar_is_high(&s); + skew = flip ^ secp256k1_scalar_is_even(&s); + secp256k1_scalar_cadd_bit(&s, 0, skew); global_sign = secp256k1_scalar_cond_negate(&s, flip); - global_sign *= not_neg_one * 2 - 1; - skew = 1 << bit; /* 4 */ u_last = secp256k1_scalar_shr_int(&s, w); @@ -214,42 +210,22 @@ static void secp256k1_ecmult_const(secp256k1_gej *r, const secp256k1_ge *a, cons } } - secp256k1_fe_mul(&r->z, &r->z, &Z); - { /* Correct for wNAF skew */ - secp256k1_ge correction = *a; - secp256k1_ge_storage correction_1_stor; - secp256k1_ge_storage correction_lam_stor; - secp256k1_ge_storage a2_stor; secp256k1_gej tmpj; - secp256k1_gej_set_ge(&tmpj, &correction); - secp256k1_gej_double_var(&tmpj, &tmpj, NULL); - secp256k1_ge_set_gej(&correction, &tmpj); - secp256k1_ge_to_storage(&correction_1_stor, a); - if (size > 128) { - secp256k1_ge_to_storage(&correction_lam_stor, a); - } - secp256k1_ge_to_storage(&a2_stor, &correction); - /* For odd numbers this is 2a (so replace it), for even ones a (so no-op) */ - secp256k1_ge_storage_cmov(&correction_1_stor, &a2_stor, skew_1 == 2); - if (size > 128) { - secp256k1_ge_storage_cmov(&correction_lam_stor, &a2_stor, skew_lam == 2); - } - - /* Apply the correction */ - secp256k1_ge_from_storage(&correction, &correction_1_stor); - secp256k1_ge_neg(&correction, &correction); - secp256k1_gej_add_ge(r, r, &correction); + secp256k1_ge_neg(&tmpa, &pre_a[0]); + secp256k1_gej_add_ge(&tmpj, r, &tmpa); + secp256k1_gej_cmov(r, &tmpj, skew_1); if (size > 128) { - secp256k1_ge_from_storage(&correction, &correction_lam_stor); - secp256k1_ge_neg(&correction, &correction); - secp256k1_ge_mul_lambda(&correction, &correction); - secp256k1_gej_add_ge(r, r, &correction); + secp256k1_ge_neg(&tmpa, &pre_a_lam[0]); + secp256k1_gej_add_ge(&tmpj, r, &tmpa); + secp256k1_gej_cmov(r, &tmpj, skew_lam); } } + + secp256k1_fe_mul(&r->z, &r->z, &Z); } #endif /* SECP256K1_ECMULT_CONST_IMPL_H */ diff --git a/src/secp256k1/src/ecmult_gen_prec.h b/src/secp256k1/src/ecmult_gen_compute_table.h index 0cfcde9b79..e577158d92 100644 --- a/src/secp256k1/src/ecmult_gen_prec.h +++ b/src/secp256k1/src/ecmult_gen_compute_table.h @@ -4,11 +4,11 @@ * file COPYING or https://www.opensource.org/licenses/mit-license.php.* ***********************************************************************/ -#ifndef SECP256K1_ECMULT_GEN_PREC_H -#define SECP256K1_ECMULT_GEN_PREC_H +#ifndef SECP256K1_ECMULT_GEN_COMPUTE_TABLE_H +#define SECP256K1_ECMULT_GEN_COMPUTE_TABLE_H #include "ecmult_gen.h" -static void secp256k1_ecmult_gen_create_prec_table(secp256k1_ge_storage* table, const secp256k1_ge* gen, int bits); +static void secp256k1_ecmult_gen_compute_table(secp256k1_ge_storage* table, const secp256k1_ge* gen, int bits); -#endif /* SECP256K1_ECMULT_GEN_PREC_H */ +#endif /* SECP256K1_ECMULT_GEN_COMPUTE_TABLE_H */ diff --git a/src/secp256k1/src/ecmult_gen_prec_impl.h b/src/secp256k1/src/ecmult_gen_compute_table_impl.h index bac76c8b13..ff6a2992dc 100644 --- a/src/secp256k1/src/ecmult_gen_prec_impl.h +++ b/src/secp256k1/src/ecmult_gen_compute_table_impl.h @@ -4,16 +4,16 @@ * file COPYING or https://www.opensource.org/licenses/mit-license.php.* ***********************************************************************/ -#ifndef SECP256K1_ECMULT_GEN_PREC_IMPL_H -#define SECP256K1_ECMULT_GEN_PREC_IMPL_H +#ifndef SECP256K1_ECMULT_GEN_COMPUTE_TABLE_IMPL_H +#define SECP256K1_ECMULT_GEN_COMPUTE_TABLE_IMPL_H -#include "ecmult_gen_prec.h" +#include "ecmult_gen_compute_table.h" #include "group_impl.h" #include "field_impl.h" #include "ecmult_gen.h" #include "util.h" -static void secp256k1_ecmult_gen_create_prec_table(secp256k1_ge_storage* table, const secp256k1_ge* gen, int bits) { +static void secp256k1_ecmult_gen_compute_table(secp256k1_ge_storage* table, const secp256k1_ge* gen, int bits) { int g = ECMULT_GEN_PREC_G(bits); int n = ECMULT_GEN_PREC_N(bits); @@ -78,4 +78,4 @@ static void secp256k1_ecmult_gen_create_prec_table(secp256k1_ge_storage* table, free(prec); } -#endif /* SECP256K1_ECMULT_GEN_PREC_IMPL_H */ +#endif /* SECP256K1_ECMULT_GEN_COMPUTE_TABLE_IMPL_H */ diff --git a/src/secp256k1/src/ecmult_gen_impl.h b/src/secp256k1/src/ecmult_gen_impl.h index 6a6ab9a4b5..2c8a503acc 100644 --- a/src/secp256k1/src/ecmult_gen_impl.h +++ b/src/secp256k1/src/ecmult_gen_impl.h @@ -12,7 +12,7 @@ #include "group.h" #include "ecmult_gen.h" #include "hash_impl.h" -#include "ecmult_gen_static_prec_table.h" +#include "precomputed_ecmult_gen.h" static void secp256k1_ecmult_gen_context_build(secp256k1_ecmult_gen_context *ctx) { secp256k1_ecmult_gen_blind(ctx, NULL); diff --git a/src/secp256k1/src/ecmult_impl.h b/src/secp256k1/src/ecmult_impl.h index 5bd4d4d23d..bbc820c77c 100644 --- a/src/secp256k1/src/ecmult_impl.h +++ b/src/secp256k1/src/ecmult_impl.h @@ -14,7 +14,7 @@ #include "group.h" #include "scalar.h" #include "ecmult.h" -#include "ecmult_static_pre_g.h" +#include "precomputed_ecmult.h" #if defined(EXHAUSTIVE_TEST_ORDER) /* We need to lower these values for exhaustive tests because @@ -47,7 +47,7 @@ /* The number of objects allocated on the scratch space for ecmult_multi algorithms */ #define PIPPENGER_SCRATCH_OBJECTS 6 -#define STRAUSS_SCRATCH_OBJECTS 7 +#define STRAUSS_SCRATCH_OBJECTS 5 #define PIPPENGER_MAX_BUCKET_WINDOW 12 @@ -56,14 +56,23 @@ #define ECMULT_MAX_POINTS_PER_BATCH 5000000 -/** Fill a table 'prej' with precomputed odd multiples of a. Prej will contain - * the values [1*a,3*a,...,(2*n-1)*a], so it space for n values. zr[0] will - * contain prej[0].z / a.z. The other zr[i] values = prej[i].z / prej[i-1].z. - * Prej's Z values are undefined, except for the last value. +/** Fill a table 'pre_a' with precomputed odd multiples of a. + * pre_a will contain [1*a,3*a,...,(2*n-1)*a], so it needs space for n group elements. + * zr needs space for n field elements. + * + * Although pre_a is an array of _ge rather than _gej, it actually represents elements + * in Jacobian coordinates with their z coordinates omitted. The omitted z-coordinates + * can be recovered using z and zr. Using the notation z(b) to represent the omitted + * z coordinate of b: + * - z(pre_a[n-1]) = 'z' + * - z(pre_a[i-1]) = z(pre_a[i]) / zr[i] for n > i > 0 + * + * Lastly the zr[0] value, which isn't used above, is set so that: + * - a.z = z(pre_a[0]) / zr[0] */ -static void secp256k1_ecmult_odd_multiples_table(int n, secp256k1_gej *prej, secp256k1_fe *zr, const secp256k1_gej *a) { - secp256k1_gej d; - secp256k1_ge a_ge, d_ge; +static void secp256k1_ecmult_odd_multiples_table(int n, secp256k1_ge *pre_a, secp256k1_fe *zr, secp256k1_fe *z, const secp256k1_gej *a) { + secp256k1_gej d, ai; + secp256k1_ge d_ge; int i; VERIFY_CHECK(!a->infinity); @@ -71,75 +80,74 @@ static void secp256k1_ecmult_odd_multiples_table(int n, secp256k1_gej *prej, sec secp256k1_gej_double_var(&d, a, NULL); /* - * Perform the additions on an isomorphism where 'd' is affine: drop the z coordinate - * of 'd', and scale the 1P starting value's x/y coordinates without changing its z. + * Perform the additions using an isomorphic curve Y^2 = X^3 + 7*C^6 where C := d.z. + * The isomorphism, phi, maps a secp256k1 point (x, y) to the point (x*C^2, y*C^3) on the other curve. + * In Jacobian coordinates phi maps (x, y, z) to (x*C^2, y*C^3, z) or, equivalently to (x, y, z/C). + * + * phi(x, y, z) = (x*C^2, y*C^3, z) = (x, y, z/C) + * d_ge := phi(d) = (d.x, d.y, 1) + * ai := phi(a) = (a.x*C^2, a.y*C^3, a.z) + * + * The group addition functions work correctly on these isomorphic curves. + * In particular phi(d) is easy to represent in affine coordinates under this isomorphism. + * This lets us use the faster secp256k1_gej_add_ge_var group addition function that we wouldn't be able to use otherwise. */ - d_ge.x = d.x; - d_ge.y = d.y; - d_ge.infinity = 0; - - secp256k1_ge_set_gej_zinv(&a_ge, a, &d.z); - prej[0].x = a_ge.x; - prej[0].y = a_ge.y; - prej[0].z = a->z; - prej[0].infinity = 0; + secp256k1_ge_set_xy(&d_ge, &d.x, &d.y); + secp256k1_ge_set_gej_zinv(&pre_a[0], a, &d.z); + secp256k1_gej_set_ge(&ai, &pre_a[0]); + ai.z = a->z; + /* pre_a[0] is the point (a.x*C^2, a.y*C^3, a.z*C) which is equvalent to a. + * Set zr[0] to C, which is the ratio between the omitted z(pre_a[0]) value and a.z. + */ zr[0] = d.z; + for (i = 1; i < n; i++) { - secp256k1_gej_add_ge_var(&prej[i], &prej[i-1], &d_ge, &zr[i]); + secp256k1_gej_add_ge_var(&ai, &ai, &d_ge, &zr[i]); + secp256k1_ge_set_xy(&pre_a[i], &ai.x, &ai.y); } - /* - * Each point in 'prej' has a z coordinate too small by a factor of 'd.z'. Only - * the final point's z coordinate is actually used though, so just update that. + /* Multiply the last z-coordinate by C to undo the isomorphism. + * Since the z-coordinates of the pre_a values are implied by the zr array of z-coordinate ratios, + * undoing the isomorphism here undoes the isomorphism for all pre_a values. */ - secp256k1_fe_mul(&prej[n-1].z, &prej[n-1].z, &d.z); + secp256k1_fe_mul(z, &ai.z, &d.z); } -/** Fill a table 'pre' with precomputed odd multiples of a. - * - * The resulting point set is brought to a single constant Z denominator, stores the X and Y - * coordinates as ge_storage points in pre, and stores the global Z in rz. - * It only operates on tables sized for WINDOW_A wnaf multiples. - * - * To compute a*P + b*G, we compute a table for P using this function, - * and use the precomputed table in <ecmult_static_pre_g.h> for G. - */ -static void secp256k1_ecmult_odd_multiples_table_globalz_windowa(secp256k1_ge *pre, secp256k1_fe *globalz, const secp256k1_gej *a) { - secp256k1_gej prej[ECMULT_TABLE_SIZE(WINDOW_A)]; - secp256k1_fe zr[ECMULT_TABLE_SIZE(WINDOW_A)]; +#define SECP256K1_ECMULT_TABLE_VERIFY(n,w) \ + VERIFY_CHECK(((n) & 1) == 1); \ + VERIFY_CHECK((n) >= -((1 << ((w)-1)) - 1)); \ + VERIFY_CHECK((n) <= ((1 << ((w)-1)) - 1)); - /* Compute the odd multiples in Jacobian form. */ - secp256k1_ecmult_odd_multiples_table(ECMULT_TABLE_SIZE(WINDOW_A), prej, zr, a); - /* Bring them to the same Z denominator. */ - secp256k1_ge_globalz_set_table_gej(ECMULT_TABLE_SIZE(WINDOW_A), pre, globalz, prej, zr); +SECP256K1_INLINE static void secp256k1_ecmult_table_get_ge(secp256k1_ge *r, const secp256k1_ge *pre, int n, int w) { + SECP256K1_ECMULT_TABLE_VERIFY(n,w) + if (n > 0) { + *r = pre[(n-1)/2]; + } else { + *r = pre[(-n-1)/2]; + secp256k1_fe_negate(&(r->y), &(r->y), 1); + } } -/** The following two macro retrieves a particular odd multiple from a table - * of precomputed multiples. */ -#define ECMULT_TABLE_GET_GE(r,pre,n,w) do { \ - VERIFY_CHECK(((n) & 1) == 1); \ - VERIFY_CHECK((n) >= -((1 << ((w)-1)) - 1)); \ - VERIFY_CHECK((n) <= ((1 << ((w)-1)) - 1)); \ - if ((n) > 0) { \ - *(r) = (pre)[((n)-1)/2]; \ - } else { \ - *(r) = (pre)[(-(n)-1)/2]; \ - secp256k1_fe_negate(&((r)->y), &((r)->y), 1); \ - } \ -} while(0) - -#define ECMULT_TABLE_GET_GE_STORAGE(r,pre,n,w) do { \ - VERIFY_CHECK(((n) & 1) == 1); \ - VERIFY_CHECK((n) >= -((1 << ((w)-1)) - 1)); \ - VERIFY_CHECK((n) <= ((1 << ((w)-1)) - 1)); \ - if ((n) > 0) { \ - secp256k1_ge_from_storage((r), &(pre)[((n)-1)/2]); \ - } else { \ - secp256k1_ge_from_storage((r), &(pre)[(-(n)-1)/2]); \ - secp256k1_fe_negate(&((r)->y), &((r)->y), 1); \ - } \ -} while(0) +SECP256K1_INLINE static void secp256k1_ecmult_table_get_ge_lambda(secp256k1_ge *r, const secp256k1_ge *pre, const secp256k1_fe *x, int n, int w) { + SECP256K1_ECMULT_TABLE_VERIFY(n,w) + if (n > 0) { + secp256k1_ge_set_xy(r, &x[(n-1)/2], &pre[(n-1)/2].y); + } else { + secp256k1_ge_set_xy(r, &x[(-n-1)/2], &pre[(-n-1)/2].y); + secp256k1_fe_negate(&(r->y), &(r->y), 1); + } +} + +SECP256K1_INLINE static void secp256k1_ecmult_table_get_ge_storage(secp256k1_ge *r, const secp256k1_ge_storage *pre, int n, int w) { + SECP256K1_ECMULT_TABLE_VERIFY(n,w) + if (n > 0) { + secp256k1_ge_from_storage(r, &pre[(n-1)/2]); + } else { + secp256k1_ge_from_storage(r, &pre[(-n-1)/2]); + secp256k1_fe_negate(&(r->y), &(r->y), 1); + } +} /** Convert a number to WNAF notation. The number becomes represented by sum(2^i * wnaf[i], i=0..bits), * with the following guarantees: @@ -201,19 +209,16 @@ static int secp256k1_ecmult_wnaf(int *wnaf, int len, const secp256k1_scalar *a, } struct secp256k1_strauss_point_state { - secp256k1_scalar na_1, na_lam; int wnaf_na_1[129]; int wnaf_na_lam[129]; int bits_na_1; int bits_na_lam; - size_t input_pos; }; struct secp256k1_strauss_state { - secp256k1_gej* prej; - secp256k1_fe* zr; + /* aux is used to hold z-ratios, and then used to hold pre_a[i].x * BETA values. */ + secp256k1_fe* aux; secp256k1_ge* pre_a; - secp256k1_ge* pre_a_lam; struct secp256k1_strauss_point_state* ps; }; @@ -231,17 +236,19 @@ static void secp256k1_ecmult_strauss_wnaf(const struct secp256k1_strauss_state * size_t np; size_t no = 0; + secp256k1_fe_set_int(&Z, 1); for (np = 0; np < num; ++np) { + secp256k1_gej tmp; + secp256k1_scalar na_1, na_lam; if (secp256k1_scalar_is_zero(&na[np]) || secp256k1_gej_is_infinity(&a[np])) { continue; } - state->ps[no].input_pos = np; /* split na into na_1 and na_lam (where na = na_1 + na_lam*lambda, and na_1 and na_lam are ~128 bit) */ - secp256k1_scalar_split_lambda(&state->ps[no].na_1, &state->ps[no].na_lam, &na[np]); + secp256k1_scalar_split_lambda(&na_1, &na_lam, &na[np]); /* build wnaf representation for na_1 and na_lam. */ - state->ps[no].bits_na_1 = secp256k1_ecmult_wnaf(state->ps[no].wnaf_na_1, 129, &state->ps[no].na_1, WINDOW_A); - state->ps[no].bits_na_lam = secp256k1_ecmult_wnaf(state->ps[no].wnaf_na_lam, 129, &state->ps[no].na_lam, WINDOW_A); + state->ps[no].bits_na_1 = secp256k1_ecmult_wnaf(state->ps[no].wnaf_na_1, 129, &na_1, WINDOW_A); + state->ps[no].bits_na_lam = secp256k1_ecmult_wnaf(state->ps[no].wnaf_na_lam, 129, &na_lam, WINDOW_A); VERIFY_CHECK(state->ps[no].bits_na_1 <= 129); VERIFY_CHECK(state->ps[no].bits_na_lam <= 129); if (state->ps[no].bits_na_1 > bits) { @@ -250,40 +257,36 @@ static void secp256k1_ecmult_strauss_wnaf(const struct secp256k1_strauss_state * if (state->ps[no].bits_na_lam > bits) { bits = state->ps[no].bits_na_lam; } - ++no; - } - /* Calculate odd multiples of a. - * All multiples are brought to the same Z 'denominator', which is stored - * in Z. Due to secp256k1' isomorphism we can do all operations pretending - * that the Z coordinate was 1, use affine addition formulae, and correct - * the Z coordinate of the result once at the end. - * The exception is the precomputed G table points, which are actually - * affine. Compared to the base used for other points, they have a Z ratio - * of 1/Z, so we can use secp256k1_gej_add_zinv_var, which uses the same - * isomorphism to efficiently add with a known Z inverse. - */ - if (no > 0) { - /* Compute the odd multiples in Jacobian form. */ - secp256k1_ecmult_odd_multiples_table(ECMULT_TABLE_SIZE(WINDOW_A), state->prej, state->zr, &a[state->ps[0].input_pos]); - for (np = 1; np < no; ++np) { - secp256k1_gej tmp = a[state->ps[np].input_pos]; + /* Calculate odd multiples of a. + * All multiples are brought to the same Z 'denominator', which is stored + * in Z. Due to secp256k1' isomorphism we can do all operations pretending + * that the Z coordinate was 1, use affine addition formulae, and correct + * the Z coordinate of the result once at the end. + * The exception is the precomputed G table points, which are actually + * affine. Compared to the base used for other points, they have a Z ratio + * of 1/Z, so we can use secp256k1_gej_add_zinv_var, which uses the same + * isomorphism to efficiently add with a known Z inverse. + */ + tmp = a[np]; + if (no) { #ifdef VERIFY - secp256k1_fe_normalize_var(&(state->prej[(np - 1) * ECMULT_TABLE_SIZE(WINDOW_A) + ECMULT_TABLE_SIZE(WINDOW_A) - 1].z)); + secp256k1_fe_normalize_var(&Z); #endif - secp256k1_gej_rescale(&tmp, &(state->prej[(np - 1) * ECMULT_TABLE_SIZE(WINDOW_A) + ECMULT_TABLE_SIZE(WINDOW_A) - 1].z)); - secp256k1_ecmult_odd_multiples_table(ECMULT_TABLE_SIZE(WINDOW_A), state->prej + np * ECMULT_TABLE_SIZE(WINDOW_A), state->zr + np * ECMULT_TABLE_SIZE(WINDOW_A), &tmp); - secp256k1_fe_mul(state->zr + np * ECMULT_TABLE_SIZE(WINDOW_A), state->zr + np * ECMULT_TABLE_SIZE(WINDOW_A), &(a[state->ps[np].input_pos].z)); + secp256k1_gej_rescale(&tmp, &Z); } - /* Bring them to the same Z denominator. */ - secp256k1_ge_globalz_set_table_gej(ECMULT_TABLE_SIZE(WINDOW_A) * no, state->pre_a, &Z, state->prej, state->zr); - } else { - secp256k1_fe_set_int(&Z, 1); + secp256k1_ecmult_odd_multiples_table(ECMULT_TABLE_SIZE(WINDOW_A), state->pre_a + no * ECMULT_TABLE_SIZE(WINDOW_A), state->aux + no * ECMULT_TABLE_SIZE(WINDOW_A), &Z, &tmp); + if (no) secp256k1_fe_mul(state->aux + no * ECMULT_TABLE_SIZE(WINDOW_A), state->aux + no * ECMULT_TABLE_SIZE(WINDOW_A), &(a[np].z)); + + ++no; } + /* Bring them to the same Z denominator. */ + secp256k1_ge_table_set_globalz(ECMULT_TABLE_SIZE(WINDOW_A) * no, state->pre_a, state->aux); + for (np = 0; np < no; ++np) { for (i = 0; i < ECMULT_TABLE_SIZE(WINDOW_A); i++) { - secp256k1_ge_mul_lambda(&state->pre_a_lam[np * ECMULT_TABLE_SIZE(WINDOW_A) + i], &state->pre_a[np * ECMULT_TABLE_SIZE(WINDOW_A) + i]); + secp256k1_fe_mul(&state->aux[np * ECMULT_TABLE_SIZE(WINDOW_A) + i], &state->pre_a[np * ECMULT_TABLE_SIZE(WINDOW_A) + i].x, &secp256k1_const_beta); } } @@ -309,20 +312,20 @@ static void secp256k1_ecmult_strauss_wnaf(const struct secp256k1_strauss_state * secp256k1_gej_double_var(r, r, NULL); for (np = 0; np < no; ++np) { if (i < state->ps[np].bits_na_1 && (n = state->ps[np].wnaf_na_1[i])) { - ECMULT_TABLE_GET_GE(&tmpa, state->pre_a + np * ECMULT_TABLE_SIZE(WINDOW_A), n, WINDOW_A); + secp256k1_ecmult_table_get_ge(&tmpa, state->pre_a + np * ECMULT_TABLE_SIZE(WINDOW_A), n, WINDOW_A); secp256k1_gej_add_ge_var(r, r, &tmpa, NULL); } if (i < state->ps[np].bits_na_lam && (n = state->ps[np].wnaf_na_lam[i])) { - ECMULT_TABLE_GET_GE(&tmpa, state->pre_a_lam + np * ECMULT_TABLE_SIZE(WINDOW_A), n, WINDOW_A); + secp256k1_ecmult_table_get_ge_lambda(&tmpa, state->pre_a + np * ECMULT_TABLE_SIZE(WINDOW_A), state->aux + np * ECMULT_TABLE_SIZE(WINDOW_A), n, WINDOW_A); secp256k1_gej_add_ge_var(r, r, &tmpa, NULL); } } if (i < bits_ng_1 && (n = wnaf_ng_1[i])) { - ECMULT_TABLE_GET_GE_STORAGE(&tmpa, secp256k1_pre_g, n, WINDOW_G); + secp256k1_ecmult_table_get_ge_storage(&tmpa, secp256k1_pre_g, n, WINDOW_G); secp256k1_gej_add_zinv_var(r, r, &tmpa, &Z); } if (i < bits_ng_128 && (n = wnaf_ng_128[i])) { - ECMULT_TABLE_GET_GE_STORAGE(&tmpa, secp256k1_pre_g_128, n, WINDOW_G); + secp256k1_ecmult_table_get_ge_storage(&tmpa, secp256k1_pre_g_128, n, WINDOW_G); secp256k1_gej_add_zinv_var(r, r, &tmpa, &Z); } } @@ -333,23 +336,19 @@ static void secp256k1_ecmult_strauss_wnaf(const struct secp256k1_strauss_state * } static void secp256k1_ecmult(secp256k1_gej *r, const secp256k1_gej *a, const secp256k1_scalar *na, const secp256k1_scalar *ng) { - secp256k1_gej prej[ECMULT_TABLE_SIZE(WINDOW_A)]; - secp256k1_fe zr[ECMULT_TABLE_SIZE(WINDOW_A)]; + secp256k1_fe aux[ECMULT_TABLE_SIZE(WINDOW_A)]; secp256k1_ge pre_a[ECMULT_TABLE_SIZE(WINDOW_A)]; struct secp256k1_strauss_point_state ps[1]; - secp256k1_ge pre_a_lam[ECMULT_TABLE_SIZE(WINDOW_A)]; struct secp256k1_strauss_state state; - state.prej = prej; - state.zr = zr; + state.aux = aux; state.pre_a = pre_a; - state.pre_a_lam = pre_a_lam; state.ps = ps; secp256k1_ecmult_strauss_wnaf(&state, r, 1, a, na, ng); } static size_t secp256k1_strauss_scratch_size(size_t n_points) { - static const size_t point_size = (2 * sizeof(secp256k1_ge) + sizeof(secp256k1_gej) + sizeof(secp256k1_fe)) * ECMULT_TABLE_SIZE(WINDOW_A) + sizeof(struct secp256k1_strauss_point_state) + sizeof(secp256k1_gej) + sizeof(secp256k1_scalar); + static const size_t point_size = (sizeof(secp256k1_ge) + sizeof(secp256k1_fe)) * ECMULT_TABLE_SIZE(WINDOW_A) + sizeof(struct secp256k1_strauss_point_state) + sizeof(secp256k1_gej) + sizeof(secp256k1_scalar); return n_points*point_size; } @@ -370,13 +369,11 @@ static int secp256k1_ecmult_strauss_batch(const secp256k1_callback* error_callba * constant and strauss_scratch_size accordingly. */ points = (secp256k1_gej*)secp256k1_scratch_alloc(error_callback, scratch, n_points * sizeof(secp256k1_gej)); scalars = (secp256k1_scalar*)secp256k1_scratch_alloc(error_callback, scratch, n_points * sizeof(secp256k1_scalar)); - state.prej = (secp256k1_gej*)secp256k1_scratch_alloc(error_callback, scratch, n_points * ECMULT_TABLE_SIZE(WINDOW_A) * sizeof(secp256k1_gej)); - state.zr = (secp256k1_fe*)secp256k1_scratch_alloc(error_callback, scratch, n_points * ECMULT_TABLE_SIZE(WINDOW_A) * sizeof(secp256k1_fe)); + state.aux = (secp256k1_fe*)secp256k1_scratch_alloc(error_callback, scratch, n_points * ECMULT_TABLE_SIZE(WINDOW_A) * sizeof(secp256k1_fe)); state.pre_a = (secp256k1_ge*)secp256k1_scratch_alloc(error_callback, scratch, n_points * ECMULT_TABLE_SIZE(WINDOW_A) * sizeof(secp256k1_ge)); - state.pre_a_lam = (secp256k1_ge*)secp256k1_scratch_alloc(error_callback, scratch, n_points * ECMULT_TABLE_SIZE(WINDOW_A) * sizeof(secp256k1_ge)); state.ps = (struct secp256k1_strauss_point_state*)secp256k1_scratch_alloc(error_callback, scratch, n_points * sizeof(struct secp256k1_strauss_point_state)); - if (points == NULL || scalars == NULL || state.prej == NULL || state.zr == NULL || state.pre_a == NULL || state.pre_a_lam == NULL || state.ps == NULL) { + if (points == NULL || scalars == NULL || state.aux == NULL || state.pre_a == NULL || state.ps == NULL) { secp256k1_scratch_apply_checkpoint(error_callback, scratch, scratch_checkpoint); return 0; } diff --git a/src/secp256k1/src/field.h b/src/secp256k1/src/field.h index 55679a2fc1..2584a494ee 100644 --- a/src/secp256k1/src/field.h +++ b/src/secp256k1/src/field.h @@ -32,6 +32,12 @@ #error "Please select wide multiplication implementation" #endif +static const secp256k1_fe secp256k1_fe_one = SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 1); +static const secp256k1_fe secp256k1_const_beta = SECP256K1_FE_CONST( + 0x7ae96a2bul, 0x657c0710ul, 0x6e64479eul, 0xac3434e9ul, + 0x9cf04975ul, 0x12f58995ul, 0xc1396c28ul, 0x719501eeul +); + /** Normalize a field element. This brings the field element to a canonical representation, reduces * its magnitude to 1, and reduces it modulo field size `p`. */ @@ -124,4 +130,13 @@ static void secp256k1_fe_storage_cmov(secp256k1_fe_storage *r, const secp256k1_f /** If flag is true, set *r equal to *a; otherwise leave it. Constant-time. Both *r and *a must be initialized.*/ static void secp256k1_fe_cmov(secp256k1_fe *r, const secp256k1_fe *a, int flag); +/** Halves the value of a field element modulo the field prime. Constant-time. + * For an input magnitude 'm', the output magnitude is set to 'floor(m/2) + 1'. + * The output is not guaranteed to be normalized, regardless of the input. */ +static void secp256k1_fe_half(secp256k1_fe *r); + +/** Sets each limb of 'r' to its upper bound at magnitude 'm'. The output will also have its + * magnitude set to 'm' and is normalized if (and only if) 'm' is zero. */ +static void secp256k1_fe_get_bounds(secp256k1_fe *r, int m); + #endif /* SECP256K1_FIELD_H */ diff --git a/src/secp256k1/src/field_10x26_impl.h b/src/secp256k1/src/field_10x26_impl.h index 4363e727e7..21742bf6eb 100644 --- a/src/secp256k1/src/field_10x26_impl.h +++ b/src/secp256k1/src/field_10x26_impl.h @@ -11,6 +11,15 @@ #include "field.h" #include "modinv32_impl.h" +/** See the comment at the top of field_5x52_impl.h for more details. + * + * Here, we represent field elements as 10 uint32_t's in base 2^26, least significant first, + * where limbs can contain >26 bits. + * A magnitude M means: + * - 2*M*(2^22-1) is the max (inclusive) of the most significant limb + * - 2*M*(2^26-1) is the max (inclusive) of the remaining limbs + */ + #ifdef VERIFY static void secp256k1_fe_verify(const secp256k1_fe *a) { const uint32_t *d = a->n; @@ -40,6 +49,26 @@ static void secp256k1_fe_verify(const secp256k1_fe *a) { } #endif +static void secp256k1_fe_get_bounds(secp256k1_fe *r, int m) { + VERIFY_CHECK(m >= 0); + VERIFY_CHECK(m <= 2048); + r->n[0] = 0x3FFFFFFUL * 2 * m; + r->n[1] = 0x3FFFFFFUL * 2 * m; + r->n[2] = 0x3FFFFFFUL * 2 * m; + r->n[3] = 0x3FFFFFFUL * 2 * m; + r->n[4] = 0x3FFFFFFUL * 2 * m; + r->n[5] = 0x3FFFFFFUL * 2 * m; + r->n[6] = 0x3FFFFFFUL * 2 * m; + r->n[7] = 0x3FFFFFFUL * 2 * m; + r->n[8] = 0x3FFFFFFUL * 2 * m; + r->n[9] = 0x03FFFFFUL * 2 * m; +#ifdef VERIFY + r->magnitude = m; + r->normalized = (m == 0); + secp256k1_fe_verify(r); +#endif +} + static void secp256k1_fe_normalize(secp256k1_fe *r) { uint32_t t0 = r->n[0], t1 = r->n[1], t2 = r->n[2], t3 = r->n[3], t4 = r->n[4], t5 = r->n[5], t6 = r->n[6], t7 = r->n[7], t8 = r->n[8], t9 = r->n[9]; @@ -391,6 +420,10 @@ SECP256K1_INLINE static void secp256k1_fe_negate(secp256k1_fe *r, const secp256k #ifdef VERIFY VERIFY_CHECK(a->magnitude <= m); secp256k1_fe_verify(a); + VERIFY_CHECK(0x3FFFC2FUL * 2 * (m + 1) >= 0x3FFFFFFUL * 2 * m); + VERIFY_CHECK(0x3FFFFBFUL * 2 * (m + 1) >= 0x3FFFFFFUL * 2 * m); + VERIFY_CHECK(0x3FFFFFFUL * 2 * (m + 1) >= 0x3FFFFFFUL * 2 * m); + VERIFY_CHECK(0x03FFFFFUL * 2 * (m + 1) >= 0x03FFFFFUL * 2 * m); #endif r->n[0] = 0x3FFFC2FUL * 2 * (m + 1) - a->n[0]; r->n[1] = 0x3FFFFBFUL * 2 * (m + 1) - a->n[1]; @@ -1120,6 +1153,82 @@ static SECP256K1_INLINE void secp256k1_fe_cmov(secp256k1_fe *r, const secp256k1_ #endif } +static SECP256K1_INLINE void secp256k1_fe_half(secp256k1_fe *r) { + uint32_t t0 = r->n[0], t1 = r->n[1], t2 = r->n[2], t3 = r->n[3], t4 = r->n[4], + t5 = r->n[5], t6 = r->n[6], t7 = r->n[7], t8 = r->n[8], t9 = r->n[9]; + uint32_t one = (uint32_t)1; + uint32_t mask = -(t0 & one) >> 6; + +#ifdef VERIFY + secp256k1_fe_verify(r); + VERIFY_CHECK(r->magnitude < 32); +#endif + + /* Bounds analysis (over the rationals). + * + * Let m = r->magnitude + * C = 0x3FFFFFFUL * 2 + * D = 0x03FFFFFUL * 2 + * + * Initial bounds: t0..t8 <= C * m + * t9 <= D * m + */ + + t0 += 0x3FFFC2FUL & mask; + t1 += 0x3FFFFBFUL & mask; + t2 += mask; + t3 += mask; + t4 += mask; + t5 += mask; + t6 += mask; + t7 += mask; + t8 += mask; + t9 += mask >> 4; + + VERIFY_CHECK((t0 & one) == 0); + + /* t0..t8: added <= C/2 + * t9: added <= D/2 + * + * Current bounds: t0..t8 <= C * (m + 1/2) + * t9 <= D * (m + 1/2) + */ + + r->n[0] = (t0 >> 1) + ((t1 & one) << 25); + r->n[1] = (t1 >> 1) + ((t2 & one) << 25); + r->n[2] = (t2 >> 1) + ((t3 & one) << 25); + r->n[3] = (t3 >> 1) + ((t4 & one) << 25); + r->n[4] = (t4 >> 1) + ((t5 & one) << 25); + r->n[5] = (t5 >> 1) + ((t6 & one) << 25); + r->n[6] = (t6 >> 1) + ((t7 & one) << 25); + r->n[7] = (t7 >> 1) + ((t8 & one) << 25); + r->n[8] = (t8 >> 1) + ((t9 & one) << 25); + r->n[9] = (t9 >> 1); + + /* t0..t8: shifted right and added <= C/4 + 1/2 + * t9: shifted right + * + * Current bounds: t0..t8 <= C * (m/2 + 1/2) + * t9 <= D * (m/2 + 1/4) + */ + +#ifdef VERIFY + /* Therefore the output magnitude (M) has to be set such that: + * t0..t8: C * M >= C * (m/2 + 1/2) + * t9: D * M >= D * (m/2 + 1/4) + * + * It suffices for all limbs that, for any input magnitude m: + * M >= m/2 + 1/2 + * + * and since we want the smallest such integer value for M: + * M == floor(m/2) + 1 + */ + r->magnitude = (r->magnitude >> 1) + 1; + r->normalized = 0; + secp256k1_fe_verify(r); +#endif +} + static SECP256K1_INLINE void secp256k1_fe_storage_cmov(secp256k1_fe_storage *r, const secp256k1_fe_storage *a, int flag) { uint32_t mask0, mask1; VG_CHECK_VERIFY(r->n, sizeof(r->n)); diff --git a/src/secp256k1/src/field_5x52_impl.h b/src/secp256k1/src/field_5x52_impl.h index b56bdd1353..6bd202f587 100644 --- a/src/secp256k1/src/field_5x52_impl.h +++ b/src/secp256k1/src/field_5x52_impl.h @@ -22,11 +22,18 @@ #endif /** Implements arithmetic modulo FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE FFFFFC2F, - * represented as 5 uint64_t's in base 2^52. The values are allowed to contain >52 each. In particular, - * each FieldElem has a 'magnitude' associated with it. Internally, a magnitude M means each element - * is at most M*(2^53-1), except the most significant one, which is limited to M*(2^49-1). All operations - * accept any input with magnitude at most M, and have different rules for propagating magnitude to their - * output. + * represented as 5 uint64_t's in base 2^52, least significant first. Note that the limbs are allowed to + * contain >52 bits each. + * + * Each field element has a 'magnitude' associated with it. Internally, a magnitude M means: + * - 2*M*(2^48-1) is the max (inclusive) of the most significant limb + * - 2*M*(2^52-1) is the max (inclusive) of the remaining limbs + * + * Operations have different rules for propagating magnitude to their outputs. If an operation takes a + * magnitude M as a parameter, that means the magnitude of input field elements can be at most M (inclusive). + * + * Each field element also has a 'normalized' flag. A field element is normalized if its magnitude is either + * 0 or 1, and its value is already reduced modulo the order of the field. */ #ifdef VERIFY @@ -51,6 +58,21 @@ static void secp256k1_fe_verify(const secp256k1_fe *a) { } #endif +static void secp256k1_fe_get_bounds(secp256k1_fe *r, int m) { + VERIFY_CHECK(m >= 0); + VERIFY_CHECK(m <= 2048); + r->n[0] = 0xFFFFFFFFFFFFFULL * 2 * m; + r->n[1] = 0xFFFFFFFFFFFFFULL * 2 * m; + r->n[2] = 0xFFFFFFFFFFFFFULL * 2 * m; + r->n[3] = 0xFFFFFFFFFFFFFULL * 2 * m; + r->n[4] = 0x0FFFFFFFFFFFFULL * 2 * m; +#ifdef VERIFY + r->magnitude = m; + r->normalized = (m == 0); + secp256k1_fe_verify(r); +#endif +} + static void secp256k1_fe_normalize(secp256k1_fe *r) { uint64_t t0 = r->n[0], t1 = r->n[1], t2 = r->n[2], t3 = r->n[3], t4 = r->n[4]; @@ -377,6 +399,9 @@ SECP256K1_INLINE static void secp256k1_fe_negate(secp256k1_fe *r, const secp256k #ifdef VERIFY VERIFY_CHECK(a->magnitude <= m); secp256k1_fe_verify(a); + VERIFY_CHECK(0xFFFFEFFFFFC2FULL * 2 * (m + 1) >= 0xFFFFFFFFFFFFFULL * 2 * m); + VERIFY_CHECK(0xFFFFFFFFFFFFFULL * 2 * (m + 1) >= 0xFFFFFFFFFFFFFULL * 2 * m); + VERIFY_CHECK(0x0FFFFFFFFFFFFULL * 2 * (m + 1) >= 0x0FFFFFFFFFFFFULL * 2 * m); #endif r->n[0] = 0xFFFFEFFFFFC2FULL * 2 * (m + 1) - a->n[0]; r->n[1] = 0xFFFFFFFFFFFFFULL * 2 * (m + 1) - a->n[1]; @@ -467,6 +492,71 @@ static SECP256K1_INLINE void secp256k1_fe_cmov(secp256k1_fe *r, const secp256k1_ #endif } +static SECP256K1_INLINE void secp256k1_fe_half(secp256k1_fe *r) { + uint64_t t0 = r->n[0], t1 = r->n[1], t2 = r->n[2], t3 = r->n[3], t4 = r->n[4]; + uint64_t one = (uint64_t)1; + uint64_t mask = -(t0 & one) >> 12; + +#ifdef VERIFY + secp256k1_fe_verify(r); + VERIFY_CHECK(r->magnitude < 32); +#endif + + /* Bounds analysis (over the rationals). + * + * Let m = r->magnitude + * C = 0xFFFFFFFFFFFFFULL * 2 + * D = 0x0FFFFFFFFFFFFULL * 2 + * + * Initial bounds: t0..t3 <= C * m + * t4 <= D * m + */ + + t0 += 0xFFFFEFFFFFC2FULL & mask; + t1 += mask; + t2 += mask; + t3 += mask; + t4 += mask >> 4; + + VERIFY_CHECK((t0 & one) == 0); + + /* t0..t3: added <= C/2 + * t4: added <= D/2 + * + * Current bounds: t0..t3 <= C * (m + 1/2) + * t4 <= D * (m + 1/2) + */ + + r->n[0] = (t0 >> 1) + ((t1 & one) << 51); + r->n[1] = (t1 >> 1) + ((t2 & one) << 51); + r->n[2] = (t2 >> 1) + ((t3 & one) << 51); + r->n[3] = (t3 >> 1) + ((t4 & one) << 51); + r->n[4] = (t4 >> 1); + + /* t0..t3: shifted right and added <= C/4 + 1/2 + * t4: shifted right + * + * Current bounds: t0..t3 <= C * (m/2 + 1/2) + * t4 <= D * (m/2 + 1/4) + */ + +#ifdef VERIFY + /* Therefore the output magnitude (M) has to be set such that: + * t0..t3: C * M >= C * (m/2 + 1/2) + * t4: D * M >= D * (m/2 + 1/4) + * + * It suffices for all limbs that, for any input magnitude m: + * M >= m/2 + 1/2 + * + * and since we want the smallest such integer value for M: + * M == floor(m/2) + 1 + */ + r->magnitude = (r->magnitude >> 1) + 1; + r->normalized = 0; + secp256k1_fe_verify(r); +#endif +} + static SECP256K1_INLINE void secp256k1_fe_storage_cmov(secp256k1_fe_storage *r, const secp256k1_fe_storage *a, int flag) { uint64_t mask0, mask1; VG_CHECK_VERIFY(r->n, sizeof(r->n)); diff --git a/src/secp256k1/src/field_impl.h b/src/secp256k1/src/field_impl.h index 374284a1f4..0a4a04d9ac 100644 --- a/src/secp256k1/src/field_impl.h +++ b/src/secp256k1/src/field_impl.h @@ -135,6 +135,4 @@ static int secp256k1_fe_sqrt(secp256k1_fe *r, const secp256k1_fe *a) { return secp256k1_fe_equal(&t1, a); } -static const secp256k1_fe secp256k1_fe_one = SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 1); - #endif /* SECP256K1_FIELD_IMPL_H */ diff --git a/src/secp256k1/src/gen_ecmult_static_pre_g.c b/src/secp256k1/src/gen_ecmult_static_pre_g.c deleted file mode 100644 index ba1d1f17d7..0000000000 --- a/src/secp256k1/src/gen_ecmult_static_pre_g.c +++ /dev/null @@ -1,131 +0,0 @@ -/***************************************************************************************************** - * Copyright (c) 2013, 2014, 2017, 2021 Pieter Wuille, Andrew Poelstra, Jonas Nick, Russell O'Connor * - * Distributed under the MIT software license, see the accompanying * - * file COPYING or https://www.opensource.org/licenses/mit-license.php. * - *****************************************************************************************************/ - -#include <inttypes.h> -#include <stdio.h> - -/* Autotools creates libsecp256k1-config.h, of which ECMULT_WINDOW_SIZE is needed. - ifndef guard so downstream users can define their own if they do not use autotools. */ -#if !defined(ECMULT_WINDOW_SIZE) -#include "libsecp256k1-config.h" -#endif - -#include "../include/secp256k1.h" -#include "assumptions.h" -#include "util.h" -#include "field_impl.h" -#include "group_impl.h" -#include "ecmult.h" - -void print_table(FILE *fp, const char *name, int window_g, const secp256k1_gej *gen, int with_conditionals) { - static secp256k1_gej gj; - static secp256k1_ge ge, dgen; - static secp256k1_ge_storage ges; - int j; - int i; - - gj = *gen; - secp256k1_ge_set_gej_var(&ge, &gj); - secp256k1_ge_to_storage(&ges, &ge); - - fprintf(fp, "static const secp256k1_ge_storage %s[ECMULT_TABLE_SIZE(WINDOW_G)] = {\n", name); - fprintf(fp, " S(%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32 - ",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32")\n", - SECP256K1_GE_STORAGE_CONST_GET(ges)); - - secp256k1_gej_double_var(&gj, gen, NULL); - secp256k1_ge_set_gej_var(&dgen, &gj); - - j = 1; - for(i = 3; i <= window_g; ++i) { - if (with_conditionals) { - fprintf(fp, "#if ECMULT_TABLE_SIZE(WINDOW_G) > %ld\n", ECMULT_TABLE_SIZE(i-1)); - } - for(;j < ECMULT_TABLE_SIZE(i); ++j) { - secp256k1_gej_set_ge(&gj, &ge); - secp256k1_gej_add_ge_var(&gj, &gj, &dgen, NULL); - secp256k1_ge_set_gej_var(&ge, &gj); - secp256k1_ge_to_storage(&ges, &ge); - - fprintf(fp, ",S(%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32 - ",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32")\n", - SECP256K1_GE_STORAGE_CONST_GET(ges)); - } - if (with_conditionals) { - fprintf(fp, "#endif\n"); - } - } - fprintf(fp, "};\n"); -} - -void print_two_tables(FILE *fp, int window_g, const secp256k1_ge *g, int with_conditionals) { - secp256k1_gej gj; - int i; - - secp256k1_gej_set_ge(&gj, g); - print_table(fp, "secp256k1_pre_g", window_g, &gj, with_conditionals); - for (i = 0; i < 128; ++i) { - secp256k1_gej_double_var(&gj, &gj, NULL); - } - print_table(fp, "secp256k1_pre_g_128", window_g, &gj, with_conditionals); -} - -int main(void) { - const secp256k1_ge g = SECP256K1_G; - const secp256k1_ge g_13 = SECP256K1_G_ORDER_13; - const secp256k1_ge g_199 = SECP256K1_G_ORDER_199; - const int window_g_13 = 4; - const int window_g_199 = 8; - FILE* fp; - - fp = fopen("src/ecmult_static_pre_g.h","w"); - if (fp == NULL) { - fprintf(stderr, "Could not open src/ecmult_static_pre_g.h for writing!\n"); - return -1; - } - - fprintf(fp, "/* This file was automatically generated by gen_ecmult_static_pre_g. */\n"); - fprintf(fp, "/* This file contains an array secp256k1_pre_g with odd multiples of the base point G and\n"); - fprintf(fp, " * an array secp256k1_pre_g_128 with odd multiples of 2^128*G for accelerating the computation of a*P + b*G.\n"); - fprintf(fp, " */\n"); - fprintf(fp, "#ifndef SECP256K1_ECMULT_STATIC_PRE_G_H\n"); - fprintf(fp, "#define SECP256K1_ECMULT_STATIC_PRE_G_H\n"); - fprintf(fp, "#include \"group.h\"\n"); - fprintf(fp, "#ifdef S\n"); - fprintf(fp, " #error macro identifier S already in use.\n"); - fprintf(fp, "#endif\n"); - fprintf(fp, "#define S(a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p) " - "SECP256K1_GE_STORAGE_CONST(0x##a##u,0x##b##u,0x##c##u,0x##d##u,0x##e##u,0x##f##u,0x##g##u," - "0x##h##u,0x##i##u,0x##j##u,0x##k##u,0x##l##u,0x##m##u,0x##n##u,0x##o##u,0x##p##u)\n"); - fprintf(fp, "#if ECMULT_TABLE_SIZE(ECMULT_WINDOW_SIZE) > %ld\n", ECMULT_TABLE_SIZE(ECMULT_WINDOW_SIZE)); - fprintf(fp, " #error configuration mismatch, invalid ECMULT_WINDOW_SIZE. Try deleting ecmult_static_pre_g.h before the build.\n"); - fprintf(fp, "#endif\n"); - fprintf(fp, "#if defined(EXHAUSTIVE_TEST_ORDER)\n"); - fprintf(fp, "#if EXHAUSTIVE_TEST_ORDER == 13\n"); - fprintf(fp, "#define WINDOW_G %d\n", window_g_13); - - print_two_tables(fp, window_g_13, &g_13, 0); - - fprintf(fp, "#elif EXHAUSTIVE_TEST_ORDER == 199\n"); - fprintf(fp, "#define WINDOW_G %d\n", window_g_199); - - print_two_tables(fp, window_g_199, &g_199, 0); - - fprintf(fp, "#else\n"); - fprintf(fp, " #error No known generator for the specified exhaustive test group order.\n"); - fprintf(fp, "#endif\n"); - fprintf(fp, "#else /* !defined(EXHAUSTIVE_TEST_ORDER) */\n"); - fprintf(fp, "#define WINDOW_G ECMULT_WINDOW_SIZE\n"); - - print_two_tables(fp, ECMULT_WINDOW_SIZE, &g, 1); - - fprintf(fp, "#endif\n"); - fprintf(fp, "#undef S\n"); - fprintf(fp, "#endif\n"); - fclose(fp); - - return 0; -} diff --git a/src/secp256k1/src/group.h b/src/secp256k1/src/group.h index b9cd334dae..bb7dae1cf7 100644 --- a/src/secp256k1/src/group.h +++ b/src/secp256k1/src/group.h @@ -9,7 +9,10 @@ #include "field.h" -/** A group element of the secp256k1 curve, in affine coordinates. */ +/** A group element in affine coordinates on the secp256k1 curve, + * or occasionally on an isomorphic curve of the form y^2 = x^3 + 7*t^6. + * Note: For exhaustive test mode, secp256k1 is replaced by a small subgroup of a different curve. + */ typedef struct { secp256k1_fe x; secp256k1_fe y; @@ -19,7 +22,9 @@ typedef struct { #define SECP256K1_GE_CONST(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) {SECP256K1_FE_CONST((a),(b),(c),(d),(e),(f),(g),(h)), SECP256K1_FE_CONST((i),(j),(k),(l),(m),(n),(o),(p)), 0} #define SECP256K1_GE_CONST_INFINITY {SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 0), 1} -/** A group element of the secp256k1 curve, in jacobian coordinates. */ +/** A group element of the secp256k1 curve, in jacobian coordinates. + * Note: For exhastive test mode, sepc256k1 is replaced by a small subgroup of a different curve. + */ typedef struct { secp256k1_fe x; /* actual X: x/z^2 */ secp256k1_fe y; /* actual Y: y/z^3 */ @@ -64,12 +69,24 @@ static void secp256k1_ge_set_gej_var(secp256k1_ge *r, secp256k1_gej *a); /** Set a batch of group elements equal to the inputs given in jacobian coordinates */ static void secp256k1_ge_set_all_gej_var(secp256k1_ge *r, const secp256k1_gej *a, size_t len); -/** Bring a batch inputs given in jacobian coordinates (with known z-ratios) to - * the same global z "denominator". zr must contain the known z-ratios such - * that mul(a[i].z, zr[i+1]) == a[i+1].z. zr[0] is ignored. The x and y - * coordinates of the result are stored in r, the common z coordinate is - * stored in globalz. */ -static void secp256k1_ge_globalz_set_table_gej(size_t len, secp256k1_ge *r, secp256k1_fe *globalz, const secp256k1_gej *a, const secp256k1_fe *zr); +/** Bring a batch of inputs to the same global z "denominator", based on ratios between + * (omitted) z coordinates of adjacent elements. + * + * Although the elements a[i] are _ge rather than _gej, they actually represent elements + * in Jacobian coordinates with their z coordinates omitted. + * + * Using the notation z(b) to represent the omitted z coordinate of b, the array zr of + * z coordinate ratios must satisfy zr[i] == z(a[i]) / z(a[i-1]) for 0 < 'i' < len. + * The zr[0] value is unused. + * + * This function adjusts the coordinates of 'a' in place so that for all 'i', z(a[i]) == z(a[len-1]). + * In other words, the initial value of z(a[len-1]) becomes the global z "denominator". Only the + * a[i].x and a[i].y coordinates are explicitly modified; the adjustment of the omitted z coordinate is + * implicit. + * + * The coordinates of the final element a[len-1] are not changed. + */ +static void secp256k1_ge_table_set_globalz(size_t len, secp256k1_ge *a, const secp256k1_fe *zr); /** Set a group element (affine) equal to the point at infinity. */ static void secp256k1_ge_set_infinity(secp256k1_ge *r); @@ -125,6 +142,9 @@ static void secp256k1_ge_to_storage(secp256k1_ge_storage *r, const secp256k1_ge static void secp256k1_ge_from_storage(secp256k1_ge *r, const secp256k1_ge_storage *a); /** If flag is true, set *r equal to *a; otherwise leave it. Constant-time. Both *r and *a must be initialized.*/ +static void secp256k1_gej_cmov(secp256k1_gej *r, const secp256k1_gej *a, int flag); + +/** If flag is true, set *r equal to *a; otherwise leave it. Constant-time. Both *r and *a must be initialized.*/ static void secp256k1_ge_storage_cmov(secp256k1_ge_storage *r, const secp256k1_ge_storage *a, int flag); /** Rescale a jacobian point by b which must be non-zero. Constant-time. */ diff --git a/src/secp256k1/src/group_impl.h b/src/secp256k1/src/group_impl.h index bce9fbdad5..b19b02a01f 100644 --- a/src/secp256k1/src/group_impl.h +++ b/src/secp256k1/src/group_impl.h @@ -161,27 +161,26 @@ static void secp256k1_ge_set_all_gej_var(secp256k1_ge *r, const secp256k1_gej *a } } -static void secp256k1_ge_globalz_set_table_gej(size_t len, secp256k1_ge *r, secp256k1_fe *globalz, const secp256k1_gej *a, const secp256k1_fe *zr) { +static void secp256k1_ge_table_set_globalz(size_t len, secp256k1_ge *a, const secp256k1_fe *zr) { size_t i = len - 1; secp256k1_fe zs; if (len > 0) { - /* The z of the final point gives us the "global Z" for the table. */ - r[i].x = a[i].x; - r[i].y = a[i].y; /* Ensure all y values are in weak normal form for fast negation of points */ - secp256k1_fe_normalize_weak(&r[i].y); - *globalz = a[i].z; - r[i].infinity = 0; + secp256k1_fe_normalize_weak(&a[i].y); zs = zr[i]; /* Work our way backwards, using the z-ratios to scale the x/y values. */ while (i > 0) { + secp256k1_gej tmpa; if (i != len - 1) { secp256k1_fe_mul(&zs, &zs, &zr[i]); } i--; - secp256k1_ge_set_gej_zinv(&r[i], &a[i], &zs); + tmpa.x = a[i].x; + tmpa.y = a[i].y; + tmpa.infinity = 0; + secp256k1_ge_set_gej_zinv(&a[i], &tmpa, &zs); } } } @@ -272,37 +271,35 @@ static int secp256k1_ge_is_valid_var(const secp256k1_ge *a) { } static SECP256K1_INLINE void secp256k1_gej_double(secp256k1_gej *r, const secp256k1_gej *a) { - /* Operations: 3 mul, 4 sqr, 0 normalize, 12 mul_int/add/negate. - * - * Note that there is an implementation described at - * https://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#doubling-dbl-2009-l - * which trades a multiply for a square, but in practice this is actually slower, - * mainly because it requires more normalizations. - */ - secp256k1_fe t1,t2,t3,t4; + /* Operations: 3 mul, 4 sqr, 8 add/half/mul_int/negate */ + secp256k1_fe l, s, t; r->infinity = a->infinity; - secp256k1_fe_mul(&r->z, &a->z, &a->y); - secp256k1_fe_mul_int(&r->z, 2); /* Z' = 2*Y*Z (2) */ - secp256k1_fe_sqr(&t1, &a->x); - secp256k1_fe_mul_int(&t1, 3); /* T1 = 3*X^2 (3) */ - secp256k1_fe_sqr(&t2, &t1); /* T2 = 9*X^4 (1) */ - secp256k1_fe_sqr(&t3, &a->y); - secp256k1_fe_mul_int(&t3, 2); /* T3 = 2*Y^2 (2) */ - secp256k1_fe_sqr(&t4, &t3); - secp256k1_fe_mul_int(&t4, 2); /* T4 = 8*Y^4 (2) */ - secp256k1_fe_mul(&t3, &t3, &a->x); /* T3 = 2*X*Y^2 (1) */ - r->x = t3; - secp256k1_fe_mul_int(&r->x, 4); /* X' = 8*X*Y^2 (4) */ - secp256k1_fe_negate(&r->x, &r->x, 4); /* X' = -8*X*Y^2 (5) */ - secp256k1_fe_add(&r->x, &t2); /* X' = 9*X^4 - 8*X*Y^2 (6) */ - secp256k1_fe_negate(&t2, &t2, 1); /* T2 = -9*X^4 (2) */ - secp256k1_fe_mul_int(&t3, 6); /* T3 = 12*X*Y^2 (6) */ - secp256k1_fe_add(&t3, &t2); /* T3 = 12*X*Y^2 - 9*X^4 (8) */ - secp256k1_fe_mul(&r->y, &t1, &t3); /* Y' = 36*X^3*Y^2 - 27*X^6 (1) */ - secp256k1_fe_negate(&t2, &t4, 2); /* T2 = -8*Y^4 (3) */ - secp256k1_fe_add(&r->y, &t2); /* Y' = 36*X^3*Y^2 - 27*X^6 - 8*Y^4 (4) */ + /* Formula used: + * L = (3/2) * X1^2 + * S = Y1^2 + * T = -X1*S + * X3 = L^2 + 2*T + * Y3 = -(L*(X3 + T) + S^2) + * Z3 = Y1*Z1 + */ + + secp256k1_fe_mul(&r->z, &a->z, &a->y); /* Z3 = Y1*Z1 (1) */ + secp256k1_fe_sqr(&s, &a->y); /* S = Y1^2 (1) */ + secp256k1_fe_sqr(&l, &a->x); /* L = X1^2 (1) */ + secp256k1_fe_mul_int(&l, 3); /* L = 3*X1^2 (3) */ + secp256k1_fe_half(&l); /* L = 3/2*X1^2 (2) */ + secp256k1_fe_negate(&t, &s, 1); /* T = -S (2) */ + secp256k1_fe_mul(&t, &t, &a->x); /* T = -X1*S (1) */ + secp256k1_fe_sqr(&r->x, &l); /* X3 = L^2 (1) */ + secp256k1_fe_add(&r->x, &t); /* X3 = L^2 + T (2) */ + secp256k1_fe_add(&r->x, &t); /* X3 = L^2 + 2*T (3) */ + secp256k1_fe_sqr(&s, &s); /* S' = S^2 (1) */ + secp256k1_fe_add(&t, &r->x); /* T' = X3 + T (4) */ + secp256k1_fe_mul(&r->y, &t, &l); /* Y3 = L*(X3 + T) (1) */ + secp256k1_fe_add(&r->y, &s); /* Y3 = L*(X3 + T) + S^2 (2) */ + secp256k1_fe_negate(&r->y, &r->y, 2); /* Y3 = -(L*(X3 + T) + S^2) (3) */ } static void secp256k1_gej_double_var(secp256k1_gej *r, const secp256k1_gej *a, secp256k1_fe *rzr) { @@ -327,7 +324,6 @@ static void secp256k1_gej_double_var(secp256k1_gej *r, const secp256k1_gej *a, s if (rzr != NULL) { *rzr = a->y; secp256k1_fe_normalize_weak(rzr); - secp256k1_fe_mul_int(rzr, 2); } secp256k1_gej_double(r, a); @@ -493,8 +489,7 @@ static void secp256k1_gej_add_zinv_var(secp256k1_gej *r, const secp256k1_gej *a, static void secp256k1_gej_add_ge(secp256k1_gej *r, const secp256k1_gej *a, const secp256k1_ge *b) { - /* Operations: 7 mul, 5 sqr, 4 normalize, 21 mul_int/add/negate/cmov */ - static const secp256k1_fe fe_1 = SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 1); + /* Operations: 7 mul, 5 sqr, 24 add/cmov/half/mul_int/negate/normalize_weak/normalizes_to_zero */ secp256k1_fe zz, u1, u2, s1, s2, t, tt, m, n, q, rr; secp256k1_fe m_alt, rr_alt; int infinity, degenerate; @@ -515,11 +510,11 @@ static void secp256k1_gej_add_ge(secp256k1_gej *r, const secp256k1_gej *a, const * Z = Z1*Z2 * T = U1+U2 * M = S1+S2 - * Q = T*M^2 + * Q = -T*M^2 * R = T^2-U1*U2 - * X3 = 4*(R^2-Q) - * Y3 = 4*(R*(3*Q-2*R^2)-M^4) - * Z3 = 2*M*Z + * X3 = R^2+Q + * Y3 = -(R*(2*X3+Q)+M^4)/2 + * Z3 = M*Z * (Note that the paper uses xi = Xi / Zi and yi = Yi / Zi instead.) * * This formula has the benefit of being the same for both addition @@ -583,7 +578,8 @@ static void secp256k1_gej_add_ge(secp256k1_gej *r, const secp256k1_gej *a, const * and denominator of lambda; R and M represent the explicit * expressions x1^2 + x2^2 + x1x2 and y1 + y2. */ secp256k1_fe_sqr(&n, &m_alt); /* n = Malt^2 (1) */ - secp256k1_fe_mul(&q, &n, &t); /* q = Q = T*Malt^2 (1) */ + secp256k1_fe_negate(&q, &t, 2); /* q = -T (3) */ + secp256k1_fe_mul(&q, &q, &n); /* q = Q = -T*Malt^2 (1) */ /* These two lines use the observation that either M == Malt or M == 0, * so M^3 * Malt is either Malt^4 (which is computed by squaring), or * zero (which is "computed" by cmov). So the cost is one squaring @@ -591,26 +587,21 @@ static void secp256k1_gej_add_ge(secp256k1_gej *r, const secp256k1_gej *a, const secp256k1_fe_sqr(&n, &n); secp256k1_fe_cmov(&n, &m, degenerate); /* n = M^3 * Malt (2) */ secp256k1_fe_sqr(&t, &rr_alt); /* t = Ralt^2 (1) */ - secp256k1_fe_mul(&r->z, &a->z, &m_alt); /* r->z = Malt*Z (1) */ + secp256k1_fe_mul(&r->z, &a->z, &m_alt); /* r->z = Z3 = Malt*Z (1) */ infinity = secp256k1_fe_normalizes_to_zero(&r->z) & ~a->infinity; - secp256k1_fe_mul_int(&r->z, 2); /* r->z = Z3 = 2*Malt*Z (2) */ - secp256k1_fe_negate(&q, &q, 1); /* q = -Q (2) */ - secp256k1_fe_add(&t, &q); /* t = Ralt^2-Q (3) */ - secp256k1_fe_normalize_weak(&t); - r->x = t; /* r->x = Ralt^2-Q (1) */ - secp256k1_fe_mul_int(&t, 2); /* t = 2*x3 (2) */ - secp256k1_fe_add(&t, &q); /* t = 2*x3 - Q: (4) */ - secp256k1_fe_mul(&t, &t, &rr_alt); /* t = Ralt*(2*x3 - Q) (1) */ - secp256k1_fe_add(&t, &n); /* t = Ralt*(2*x3 - Q) + M^3*Malt (3) */ - secp256k1_fe_negate(&r->y, &t, 3); /* r->y = Ralt*(Q - 2x3) - M^3*Malt (4) */ - secp256k1_fe_normalize_weak(&r->y); - secp256k1_fe_mul_int(&r->x, 4); /* r->x = X3 = 4*(Ralt^2-Q) */ - secp256k1_fe_mul_int(&r->y, 4); /* r->y = Y3 = 4*Ralt*(Q - 2x3) - 4*M^3*Malt (4) */ + secp256k1_fe_add(&t, &q); /* t = Ralt^2 + Q (2) */ + r->x = t; /* r->x = X3 = Ralt^2 + Q (2) */ + secp256k1_fe_mul_int(&t, 2); /* t = 2*X3 (4) */ + secp256k1_fe_add(&t, &q); /* t = 2*X3 + Q (5) */ + secp256k1_fe_mul(&t, &t, &rr_alt); /* t = Ralt*(2*X3 + Q) (1) */ + secp256k1_fe_add(&t, &n); /* t = Ralt*(2*X3 + Q) + M^3*Malt (3) */ + secp256k1_fe_negate(&r->y, &t, 3); /* r->y = -(Ralt*(2*X3 + Q) + M^3*Malt) (4) */ + secp256k1_fe_half(&r->y); /* r->y = Y3 = -(Ralt*(2*X3 + Q) + M^3*Malt)/2 (3) */ /** In case a->infinity == 1, replace r with (b->x, b->y, 1). */ secp256k1_fe_cmov(&r->x, &b->x, a->infinity); secp256k1_fe_cmov(&r->y, &b->y, a->infinity); - secp256k1_fe_cmov(&r->z, &fe_1, a->infinity); + secp256k1_fe_cmov(&r->z, &secp256k1_fe_one, a->infinity); r->infinity = infinity; } @@ -642,18 +633,22 @@ static void secp256k1_ge_from_storage(secp256k1_ge *r, const secp256k1_ge_storag r->infinity = 0; } +static SECP256K1_INLINE void secp256k1_gej_cmov(secp256k1_gej *r, const secp256k1_gej *a, int flag) { + secp256k1_fe_cmov(&r->x, &a->x, flag); + secp256k1_fe_cmov(&r->y, &a->y, flag); + secp256k1_fe_cmov(&r->z, &a->z, flag); + + r->infinity ^= (r->infinity ^ a->infinity) & flag; +} + static SECP256K1_INLINE void secp256k1_ge_storage_cmov(secp256k1_ge_storage *r, const secp256k1_ge_storage *a, int flag) { secp256k1_fe_storage_cmov(&r->x, &a->x, flag); secp256k1_fe_storage_cmov(&r->y, &a->y, flag); } static void secp256k1_ge_mul_lambda(secp256k1_ge *r, const secp256k1_ge *a) { - static const secp256k1_fe beta = SECP256K1_FE_CONST( - 0x7ae96a2bul, 0x657c0710ul, 0x6e64479eul, 0xac3434e9ul, - 0x9cf04975ul, 0x12f58995ul, 0xc1396c28ul, 0x719501eeul - ); *r = *a; - secp256k1_fe_mul(&r->x, &r->x, &beta); + secp256k1_fe_mul(&r->x, &r->x, &secp256k1_const_beta); } static int secp256k1_ge_is_in_correct_subgroup(const secp256k1_ge* ge) { diff --git a/src/secp256k1/src/hash.h b/src/secp256k1/src/hash.h index 0947a09694..4e0384cfbf 100644 --- a/src/secp256k1/src/hash.h +++ b/src/secp256k1/src/hash.h @@ -12,8 +12,8 @@ typedef struct { uint32_t s[8]; - uint32_t buf[16]; /* In big endian */ - size_t bytes; + unsigned char buf[64]; + uint64_t bytes; } secp256k1_sha256; static void secp256k1_sha256_initialize(secp256k1_sha256 *hash); diff --git a/src/secp256k1/src/hash_impl.h b/src/secp256k1/src/hash_impl.h index f8cd3a1634..0991fe7838 100644 --- a/src/secp256k1/src/hash_impl.h +++ b/src/secp256k1/src/hash_impl.h @@ -28,12 +28,6 @@ (h) = t1 + t2; \ } while(0) -#if defined(SECP256K1_BIG_ENDIAN) -#define BE32(x) (x) -#elif defined(SECP256K1_LITTLE_ENDIAN) -#define BE32(p) ((((p) & 0xFF) << 24) | (((p) & 0xFF00) << 8) | (((p) & 0xFF0000) >> 8) | (((p) & 0xFF000000) >> 24)) -#endif - static void secp256k1_sha256_initialize(secp256k1_sha256 *hash) { hash->s[0] = 0x6a09e667ul; hash->s[1] = 0xbb67ae85ul; @@ -47,26 +41,26 @@ static void secp256k1_sha256_initialize(secp256k1_sha256 *hash) { } /** Perform one SHA-256 transformation, processing 16 big endian 32-bit words. */ -static void secp256k1_sha256_transform(uint32_t* s, const uint32_t* chunk) { +static void secp256k1_sha256_transform(uint32_t* s, const unsigned char* buf) { uint32_t a = s[0], b = s[1], c = s[2], d = s[3], e = s[4], f = s[5], g = s[6], h = s[7]; uint32_t w0, w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11, w12, w13, w14, w15; - Round(a, b, c, d, e, f, g, h, 0x428a2f98, w0 = BE32(chunk[0])); - Round(h, a, b, c, d, e, f, g, 0x71374491, w1 = BE32(chunk[1])); - Round(g, h, a, b, c, d, e, f, 0xb5c0fbcf, w2 = BE32(chunk[2])); - Round(f, g, h, a, b, c, d, e, 0xe9b5dba5, w3 = BE32(chunk[3])); - Round(e, f, g, h, a, b, c, d, 0x3956c25b, w4 = BE32(chunk[4])); - Round(d, e, f, g, h, a, b, c, 0x59f111f1, w5 = BE32(chunk[5])); - Round(c, d, e, f, g, h, a, b, 0x923f82a4, w6 = BE32(chunk[6])); - Round(b, c, d, e, f, g, h, a, 0xab1c5ed5, w7 = BE32(chunk[7])); - Round(a, b, c, d, e, f, g, h, 0xd807aa98, w8 = BE32(chunk[8])); - Round(h, a, b, c, d, e, f, g, 0x12835b01, w9 = BE32(chunk[9])); - Round(g, h, a, b, c, d, e, f, 0x243185be, w10 = BE32(chunk[10])); - Round(f, g, h, a, b, c, d, e, 0x550c7dc3, w11 = BE32(chunk[11])); - Round(e, f, g, h, a, b, c, d, 0x72be5d74, w12 = BE32(chunk[12])); - Round(d, e, f, g, h, a, b, c, 0x80deb1fe, w13 = BE32(chunk[13])); - Round(c, d, e, f, g, h, a, b, 0x9bdc06a7, w14 = BE32(chunk[14])); - Round(b, c, d, e, f, g, h, a, 0xc19bf174, w15 = BE32(chunk[15])); + Round(a, b, c, d, e, f, g, h, 0x428a2f98, w0 = secp256k1_read_be32(&buf[0])); + Round(h, a, b, c, d, e, f, g, 0x71374491, w1 = secp256k1_read_be32(&buf[4])); + Round(g, h, a, b, c, d, e, f, 0xb5c0fbcf, w2 = secp256k1_read_be32(&buf[8])); + Round(f, g, h, a, b, c, d, e, 0xe9b5dba5, w3 = secp256k1_read_be32(&buf[12])); + Round(e, f, g, h, a, b, c, d, 0x3956c25b, w4 = secp256k1_read_be32(&buf[16])); + Round(d, e, f, g, h, a, b, c, 0x59f111f1, w5 = secp256k1_read_be32(&buf[20])); + Round(c, d, e, f, g, h, a, b, 0x923f82a4, w6 = secp256k1_read_be32(&buf[24])); + Round(b, c, d, e, f, g, h, a, 0xab1c5ed5, w7 = secp256k1_read_be32(&buf[28])); + Round(a, b, c, d, e, f, g, h, 0xd807aa98, w8 = secp256k1_read_be32(&buf[32])); + Round(h, a, b, c, d, e, f, g, 0x12835b01, w9 = secp256k1_read_be32(&buf[36])); + Round(g, h, a, b, c, d, e, f, 0x243185be, w10 = secp256k1_read_be32(&buf[40])); + Round(f, g, h, a, b, c, d, e, 0x550c7dc3, w11 = secp256k1_read_be32(&buf[44])); + Round(e, f, g, h, a, b, c, d, 0x72be5d74, w12 = secp256k1_read_be32(&buf[48])); + Round(d, e, f, g, h, a, b, c, 0x80deb1fe, w13 = secp256k1_read_be32(&buf[52])); + Round(c, d, e, f, g, h, a, b, 0x9bdc06a7, w14 = secp256k1_read_be32(&buf[56])); + Round(b, c, d, e, f, g, h, a, 0xc19bf174, w15 = secp256k1_read_be32(&buf[60])); Round(a, b, c, d, e, f, g, h, 0xe49b69c1, w0 += sigma1(w14) + w9 + sigma0(w1)); Round(h, a, b, c, d, e, f, g, 0xefbe4786, w1 += sigma1(w15) + w10 + sigma0(w2)); @@ -136,7 +130,7 @@ static void secp256k1_sha256_write(secp256k1_sha256 *hash, const unsigned char * while (len >= 64 - bufsize) { /* Fill the buffer, and process it. */ size_t chunk_len = 64 - bufsize; - memcpy(((unsigned char*)hash->buf) + bufsize, data, chunk_len); + memcpy(hash->buf + bufsize, data, chunk_len); data += chunk_len; len -= chunk_len; secp256k1_sha256_transform(hash->s, hash->buf); @@ -149,19 +143,19 @@ static void secp256k1_sha256_write(secp256k1_sha256 *hash, const unsigned char * } static void secp256k1_sha256_finalize(secp256k1_sha256 *hash, unsigned char *out32) { - static const unsigned char pad[64] = {0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - uint32_t sizedesc[2]; - uint32_t out[8]; - int i = 0; - sizedesc[0] = BE32(hash->bytes >> 29); - sizedesc[1] = BE32(hash->bytes << 3); + static const unsigned char pad[64] = {0x80}; + unsigned char sizedesc[8]; + int i; + /* The maximum message size of SHA256 is 2^64-1 bits. */ + VERIFY_CHECK(hash->bytes < ((uint64_t)1 << 61)); + secp256k1_write_be32(&sizedesc[0], hash->bytes >> 29); + secp256k1_write_be32(&sizedesc[4], hash->bytes << 3); secp256k1_sha256_write(hash, pad, 1 + ((119 - (hash->bytes % 64)) % 64)); - secp256k1_sha256_write(hash, (const unsigned char*)sizedesc, 8); + secp256k1_sha256_write(hash, sizedesc, 8); for (i = 0; i < 8; i++) { - out[i] = BE32(hash->s[i]); + secp256k1_write_be32(&out32[4*i], hash->s[i]); hash->s[i] = 0; } - memcpy(out32, (const unsigned char*)out, 32); } /* Initializes a sha256 struct and writes the 64 byte string @@ -285,7 +279,6 @@ static void secp256k1_rfc6979_hmac_sha256_finalize(secp256k1_rfc6979_hmac_sha256 rng->retry = 0; } -#undef BE32 #undef Round #undef sigma1 #undef sigma0 diff --git a/src/secp256k1/src/modules/ecdh/tests_impl.h b/src/secp256k1/src/modules/ecdh/tests_impl.h index be07447a4b..10b7075c38 100644 --- a/src/secp256k1/src/modules/ecdh/tests_impl.h +++ b/src/secp256k1/src/modules/ecdh/tests_impl.h @@ -60,7 +60,7 @@ void test_ecdh_generator_basepoint(void) { s_one[31] = 1; /* Check against pubkey creation when the basepoint is the generator */ - for (i = 0; i < 100; ++i) { + for (i = 0; i < 2 * count; ++i) { secp256k1_sha256 sha; unsigned char s_b32[32]; unsigned char output_ecdh[65]; @@ -123,10 +123,43 @@ void test_bad_scalar(void) { CHECK(secp256k1_ecdh(ctx, output, &point, s_overflow, ecdh_hash_function_test_fail, NULL) == 0); } +/** Test that ECDH(sG, 1/s) == ECDH((1/s)G, s) == ECDH(G, 1) for a few random s. */ +void test_result_basepoint(void) { + secp256k1_pubkey point; + secp256k1_scalar rand; + unsigned char s[32]; + unsigned char s_inv[32]; + unsigned char out[32]; + unsigned char out_inv[32]; + unsigned char out_base[32]; + int i; + + unsigned char s_one[32] = { 0 }; + s_one[31] = 1; + CHECK(secp256k1_ec_pubkey_create(ctx, &point, s_one) == 1); + CHECK(secp256k1_ecdh(ctx, out_base, &point, s_one, NULL, NULL) == 1); + + for (i = 0; i < 2 * count; i++) { + random_scalar_order(&rand); + secp256k1_scalar_get_b32(s, &rand); + secp256k1_scalar_inverse(&rand, &rand); + secp256k1_scalar_get_b32(s_inv, &rand); + + CHECK(secp256k1_ec_pubkey_create(ctx, &point, s) == 1); + CHECK(secp256k1_ecdh(ctx, out, &point, s_inv, NULL, NULL) == 1); + CHECK(secp256k1_memcmp_var(out, out_base, 32) == 0); + + CHECK(secp256k1_ec_pubkey_create(ctx, &point, s_inv) == 1); + CHECK(secp256k1_ecdh(ctx, out_inv, &point, s, NULL, NULL) == 1); + CHECK(secp256k1_memcmp_var(out_inv, out_base, 32) == 0); + } +} + void run_ecdh_tests(void) { test_ecdh_api(); test_ecdh_generator_basepoint(); test_bad_scalar(); + test_result_basepoint(); } #endif /* SECP256K1_MODULE_ECDH_TESTS_H */ diff --git a/src/secp256k1/src/modules/schnorrsig/main_impl.h b/src/secp256k1/src/modules/schnorrsig/main_impl.h index 94e3ee414d..cd651591c4 100644 --- a/src/secp256k1/src/modules/schnorrsig/main_impl.h +++ b/src/secp256k1/src/modules/schnorrsig/main_impl.h @@ -192,11 +192,15 @@ static int secp256k1_schnorrsig_sign_internal(const secp256k1_context* ctx, unsi return ret; } -int secp256k1_schnorrsig_sign(const secp256k1_context* ctx, unsigned char *sig64, const unsigned char *msg32, const secp256k1_keypair *keypair, const unsigned char *aux_rand32) { +int secp256k1_schnorrsig_sign32(const secp256k1_context* ctx, unsigned char *sig64, const unsigned char *msg32, const secp256k1_keypair *keypair, const unsigned char *aux_rand32) { /* We cast away const from the passed aux_rand32 argument since we know the default nonce function does not modify it. */ return secp256k1_schnorrsig_sign_internal(ctx, sig64, msg32, 32, keypair, secp256k1_nonce_function_bip340, (unsigned char*)aux_rand32); } +int secp256k1_schnorrsig_sign(const secp256k1_context* ctx, unsigned char *sig64, const unsigned char *msg32, const secp256k1_keypair *keypair, const unsigned char *aux_rand32) { + return secp256k1_schnorrsig_sign32(ctx, sig64, msg32, keypair, aux_rand32); +} + int secp256k1_schnorrsig_sign_custom(const secp256k1_context* ctx, unsigned char *sig64, const unsigned char *msg, size_t msglen, const secp256k1_keypair *keypair, secp256k1_schnorrsig_extraparams *extraparams) { secp256k1_nonce_function_hardened noncefp = NULL; void *ndata = NULL; diff --git a/src/secp256k1/src/modules/schnorrsig/tests_impl.h b/src/secp256k1/src/modules/schnorrsig/tests_impl.h index 2efec8a2b9..25840b8fa7 100644 --- a/src/secp256k1/src/modules/schnorrsig/tests_impl.h +++ b/src/secp256k1/src/modules/schnorrsig/tests_impl.h @@ -87,7 +87,7 @@ void run_nonce_function_bip340_tests(void) { CHECK(nonce_function_bip340(nonce, msg, msglen, key, pk, NULL, 0, NULL) == 0); CHECK(nonce_function_bip340(nonce, msg, msglen, key, pk, algo, algolen, NULL) == 1); /* Other algo is fine */ - secp256k1_rfc6979_hmac_sha256_generate(&secp256k1_test_rng, algo, algolen); + secp256k1_testrand_bytes_test(algo, algolen); CHECK(nonce_function_bip340(nonce, msg, msglen, key, pk, algo, algolen, NULL) == 1); for (i = 0; i < count; i++) { @@ -160,21 +160,21 @@ void test_schnorrsig_api(void) { /** main test body **/ ecount = 0; - CHECK(secp256k1_schnorrsig_sign(none, sig, msg, &keypairs[0], NULL) == 1); + CHECK(secp256k1_schnorrsig_sign32(none, sig, msg, &keypairs[0], NULL) == 1); CHECK(ecount == 0); - CHECK(secp256k1_schnorrsig_sign(vrfy, sig, msg, &keypairs[0], NULL) == 1); + CHECK(secp256k1_schnorrsig_sign32(vrfy, sig, msg, &keypairs[0], NULL) == 1); CHECK(ecount == 0); - CHECK(secp256k1_schnorrsig_sign(sign, sig, msg, &keypairs[0], NULL) == 1); + CHECK(secp256k1_schnorrsig_sign32(sign, sig, msg, &keypairs[0], NULL) == 1); CHECK(ecount == 0); - CHECK(secp256k1_schnorrsig_sign(sign, NULL, msg, &keypairs[0], NULL) == 0); + CHECK(secp256k1_schnorrsig_sign32(sign, NULL, msg, &keypairs[0], NULL) == 0); CHECK(ecount == 1); - CHECK(secp256k1_schnorrsig_sign(sign, sig, NULL, &keypairs[0], NULL) == 0); + CHECK(secp256k1_schnorrsig_sign32(sign, sig, NULL, &keypairs[0], NULL) == 0); CHECK(ecount == 2); - CHECK(secp256k1_schnorrsig_sign(sign, sig, msg, NULL, NULL) == 0); + CHECK(secp256k1_schnorrsig_sign32(sign, sig, msg, NULL, NULL) == 0); CHECK(ecount == 3); - CHECK(secp256k1_schnorrsig_sign(sign, sig, msg, &invalid_keypair, NULL) == 0); + CHECK(secp256k1_schnorrsig_sign32(sign, sig, msg, &invalid_keypair, NULL) == 0); CHECK(ecount == 4); - CHECK(secp256k1_schnorrsig_sign(sttc, sig, msg, &keypairs[0], NULL) == 0); + CHECK(secp256k1_schnorrsig_sign32(sttc, sig, msg, &keypairs[0], NULL) == 0); CHECK(ecount == 5); ecount = 0; @@ -202,7 +202,7 @@ void test_schnorrsig_api(void) { CHECK(ecount == 6); ecount = 0; - CHECK(secp256k1_schnorrsig_sign(sign, sig, msg, &keypairs[0], NULL) == 1); + CHECK(secp256k1_schnorrsig_sign32(sign, sig, msg, &keypairs[0], NULL) == 1); CHECK(secp256k1_schnorrsig_verify(none, sig, msg, sizeof(msg), &pk[0]) == 1); CHECK(ecount == 0); CHECK(secp256k1_schnorrsig_verify(sign, sig, msg, sizeof(msg), &pk[0]) == 1); @@ -247,7 +247,7 @@ void test_schnorrsig_bip_vectors_check_signing(const unsigned char *sk, const un secp256k1_xonly_pubkey pk, pk_expected; CHECK(secp256k1_keypair_create(ctx, &keypair, sk)); - CHECK(secp256k1_schnorrsig_sign(ctx, sig, msg32, &keypair, aux_rand)); + CHECK(secp256k1_schnorrsig_sign32(ctx, sig, msg32, &keypair, aux_rand)); CHECK(secp256k1_memcmp_var(sig, expected_sig, 64) == 0); CHECK(secp256k1_xonly_pubkey_parse(ctx, &pk_expected, pk_serialized)); @@ -740,8 +740,11 @@ void test_schnorrsig_sign(void) { secp256k1_testrand256(aux_rand); CHECK(secp256k1_keypair_create(ctx, &keypair, sk)); CHECK(secp256k1_keypair_xonly_pub(ctx, &pk, NULL, &keypair)); - CHECK(secp256k1_schnorrsig_sign(ctx, sig, msg, &keypair, NULL) == 1); + CHECK(secp256k1_schnorrsig_sign32(ctx, sig, msg, &keypair, NULL) == 1); CHECK(secp256k1_schnorrsig_verify(ctx, sig, msg, sizeof(msg), &pk)); + /* Check that deprecated alias gives the same result */ + CHECK(secp256k1_schnorrsig_sign(ctx, sig2, msg, &keypair, NULL) == 1); + CHECK(secp256k1_memcmp_var(sig, sig2, sizeof(sig)) == 0); /* Test different nonce functions */ CHECK(secp256k1_schnorrsig_sign_custom(ctx, sig, msg, sizeof(msg), &keypair, &extraparams) == 1); @@ -764,7 +767,7 @@ void test_schnorrsig_sign(void) { extraparams.noncefp = NULL; extraparams.ndata = aux_rand; CHECK(secp256k1_schnorrsig_sign_custom(ctx, sig, msg, sizeof(msg), &keypair, &extraparams) == 1); - CHECK(secp256k1_schnorrsig_sign(ctx, sig2, msg, &keypair, extraparams.ndata) == 1); + CHECK(secp256k1_schnorrsig_sign32(ctx, sig2, msg, &keypair, extraparams.ndata) == 1); CHECK(secp256k1_memcmp_var(sig, sig2, sizeof(sig)) == 0); } @@ -787,7 +790,7 @@ void test_schnorrsig_sign_verify(void) { for (i = 0; i < N_SIGS; i++) { secp256k1_testrand256(msg[i]); - CHECK(secp256k1_schnorrsig_sign(ctx, sig[i], msg[i], &keypair, NULL)); + CHECK(secp256k1_schnorrsig_sign32(ctx, sig[i], msg[i], &keypair, NULL)); CHECK(secp256k1_schnorrsig_verify(ctx, sig[i], msg[i], sizeof(msg[i]), &pk)); } @@ -795,18 +798,18 @@ void test_schnorrsig_sign_verify(void) { /* Flip a few bits in the signature and in the message and check that * verify and verify_batch (TODO) fail */ size_t sig_idx = secp256k1_testrand_int(N_SIGS); - size_t byte_idx = secp256k1_testrand_int(32); + size_t byte_idx = secp256k1_testrand_bits(5); unsigned char xorbyte = secp256k1_testrand_int(254)+1; sig[sig_idx][byte_idx] ^= xorbyte; CHECK(!secp256k1_schnorrsig_verify(ctx, sig[sig_idx], msg[sig_idx], sizeof(msg[sig_idx]), &pk)); sig[sig_idx][byte_idx] ^= xorbyte; - byte_idx = secp256k1_testrand_int(32); + byte_idx = secp256k1_testrand_bits(5); sig[sig_idx][32+byte_idx] ^= xorbyte; CHECK(!secp256k1_schnorrsig_verify(ctx, sig[sig_idx], msg[sig_idx], sizeof(msg[sig_idx]), &pk)); sig[sig_idx][32+byte_idx] ^= xorbyte; - byte_idx = secp256k1_testrand_int(32); + byte_idx = secp256k1_testrand_bits(5); msg[sig_idx][byte_idx] ^= xorbyte; CHECK(!secp256k1_schnorrsig_verify(ctx, sig[sig_idx], msg[sig_idx], sizeof(msg[sig_idx]), &pk)); msg[sig_idx][byte_idx] ^= xorbyte; @@ -816,13 +819,13 @@ void test_schnorrsig_sign_verify(void) { } /* Test overflowing s */ - CHECK(secp256k1_schnorrsig_sign(ctx, sig[0], msg[0], &keypair, NULL)); + CHECK(secp256k1_schnorrsig_sign32(ctx, sig[0], msg[0], &keypair, NULL)); CHECK(secp256k1_schnorrsig_verify(ctx, sig[0], msg[0], sizeof(msg[0]), &pk)); memset(&sig[0][32], 0xFF, 32); CHECK(!secp256k1_schnorrsig_verify(ctx, sig[0], msg[0], sizeof(msg[0]), &pk)); /* Test negative s */ - CHECK(secp256k1_schnorrsig_sign(ctx, sig[0], msg[0], &keypair, NULL)); + CHECK(secp256k1_schnorrsig_sign32(ctx, sig[0], msg[0], &keypair, NULL)); CHECK(secp256k1_schnorrsig_verify(ctx, sig[0], msg[0], sizeof(msg[0]), &pk)); secp256k1_scalar_set_b32(&s, &sig[0][32], NULL); secp256k1_scalar_negate(&s, &s); @@ -873,7 +876,7 @@ void test_schnorrsig_taproot(void) { /* Key spend */ secp256k1_testrand256(msg); - CHECK(secp256k1_schnorrsig_sign(ctx, sig, msg, &keypair, NULL) == 1); + CHECK(secp256k1_schnorrsig_sign32(ctx, sig, msg, &keypair, NULL) == 1); /* Verify key spend */ CHECK(secp256k1_xonly_pubkey_parse(ctx, &output_pk, output_pk_bytes) == 1); CHECK(secp256k1_schnorrsig_verify(ctx, sig, msg, sizeof(msg), &output_pk) == 1); diff --git a/src/secp256k1/src/precompute_ecmult.c b/src/secp256k1/src/precompute_ecmult.c new file mode 100644 index 0000000000..5ccbcb3c57 --- /dev/null +++ b/src/secp256k1/src/precompute_ecmult.c @@ -0,0 +1,96 @@ +/***************************************************************************************************** + * Copyright (c) 2013, 2014, 2017, 2021 Pieter Wuille, Andrew Poelstra, Jonas Nick, Russell O'Connor * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or https://www.opensource.org/licenses/mit-license.php. * + *****************************************************************************************************/ + +#include <inttypes.h> +#include <stdio.h> + +/* Autotools creates libsecp256k1-config.h, of which ECMULT_WINDOW_SIZE is needed. + ifndef guard so downstream users can define their own if they do not use autotools. */ +#if !defined(ECMULT_WINDOW_SIZE) +#include "libsecp256k1-config.h" +#endif + +#include "../include/secp256k1.h" +#include "assumptions.h" +#include "util.h" +#include "field_impl.h" +#include "group_impl.h" +#include "ecmult.h" +#include "ecmult_compute_table_impl.h" + +static void print_table(FILE *fp, const char *name, int window_g, const secp256k1_ge_storage* table) { + int j; + int i; + + fprintf(fp, "const secp256k1_ge_storage %s[ECMULT_TABLE_SIZE(WINDOW_G)] = {\n", name); + fprintf(fp, " S(%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32 + ",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32")\n", + SECP256K1_GE_STORAGE_CONST_GET(table[0])); + + j = 1; + for(i = 3; i <= window_g; ++i) { + fprintf(fp, "#if WINDOW_G > %d\n", i-1); + for(;j < ECMULT_TABLE_SIZE(i); ++j) { + fprintf(fp, ",S(%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32 + ",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32")\n", + SECP256K1_GE_STORAGE_CONST_GET(table[j])); + } + fprintf(fp, "#endif\n"); + } + fprintf(fp, "};\n"); +} + +static void print_two_tables(FILE *fp, int window_g) { + secp256k1_ge_storage* table = malloc(ECMULT_TABLE_SIZE(window_g) * sizeof(secp256k1_ge_storage)); + secp256k1_ge_storage* table_128 = malloc(ECMULT_TABLE_SIZE(window_g) * sizeof(secp256k1_ge_storage)); + + secp256k1_ecmult_compute_two_tables(table, table_128, window_g, &secp256k1_ge_const_g); + + print_table(fp, "secp256k1_pre_g", window_g, table); + print_table(fp, "secp256k1_pre_g_128", window_g, table_128); + + free(table); + free(table_128); +} + +int main(void) { + /* Always compute all tables for window sizes up to 15. */ + int window_g = (ECMULT_WINDOW_SIZE < 15) ? 15 : ECMULT_WINDOW_SIZE; + FILE* fp; + + fp = fopen("src/precomputed_ecmult.c","w"); + if (fp == NULL) { + fprintf(stderr, "Could not open src/precomputed_ecmult.h for writing!\n"); + return -1; + } + + fprintf(fp, "/* This file was automatically generated by precompute_ecmult. */\n"); + fprintf(fp, "/* This file contains an array secp256k1_pre_g with odd multiples of the base point G and\n"); + fprintf(fp, " * an array secp256k1_pre_g_128 with odd multiples of 2^128*G for accelerating the computation of a*P + b*G.\n"); + fprintf(fp, " */\n"); + fprintf(fp, "#if defined HAVE_CONFIG_H\n"); + fprintf(fp, "# include \"libsecp256k1-config.h\"\n"); + fprintf(fp, "#endif\n"); + fprintf(fp, "#include \"../include/secp256k1.h\"\n"); + fprintf(fp, "#include \"group.h\"\n"); + fprintf(fp, "#include \"ecmult.h\"\n"); + fprintf(fp, "#include \"precomputed_ecmult.h\"\n"); + fprintf(fp, "#define S(a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p) SECP256K1_GE_STORAGE_CONST(0x##a##u,0x##b##u,0x##c##u,0x##d##u,0x##e##u,0x##f##u,0x##g##u,0x##h##u,0x##i##u,0x##j##u,0x##k##u,0x##l##u,0x##m##u,0x##n##u,0x##o##u,0x##p##u)\n"); + fprintf(fp, "#if ECMULT_WINDOW_SIZE > %d\n", window_g); + fprintf(fp, " #error configuration mismatch, invalid ECMULT_WINDOW_SIZE. Try deleting precomputed_ecmult.c before the build.\n"); + fprintf(fp, "#endif\n"); + fprintf(fp, "#ifdef EXHAUSTIVE_TEST_ORDER\n"); + fprintf(fp, "# error Cannot compile precomputed_ecmult.c in exhaustive test mode\n"); + fprintf(fp, "#endif /* EXHAUSTIVE_TEST_ORDER */\n"); + fprintf(fp, "#define WINDOW_G ECMULT_WINDOW_SIZE\n"); + + print_two_tables(fp, window_g); + + fprintf(fp, "#undef S\n"); + fclose(fp); + + return 0; +} diff --git a/src/secp256k1/src/gen_ecmult_gen_static_prec_table.c b/src/secp256k1/src/precompute_ecmult_gen.c index 22923df313..7c6359c402 100644 --- a/src/secp256k1/src/gen_ecmult_gen_static_prec_table.c +++ b/src/secp256k1/src/precompute_ecmult_gen.c @@ -1,8 +1,8 @@ -/*********************************************************************** - * Copyright (c) 2013, 2014, 2015 Thomas Daede, Cory Fields * - * Distributed under the MIT software license, see the accompanying * - * file COPYING or https://www.opensource.org/licenses/mit-license.php.* - ***********************************************************************/ +/********************************************************************************* + * Copyright (c) 2013, 2014, 2015, 2021 Thomas Daede, Cory Fields, Pieter Wuille * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or https://www.opensource.org/licenses/mit-license.php. * + *********************************************************************************/ #include <inttypes.h> #include <stdio.h> @@ -12,10 +12,10 @@ #include "util.h" #include "group.h" #include "ecmult_gen.h" -#include "ecmult_gen_prec_impl.h" +#include "ecmult_gen_compute_table_impl.h" int main(int argc, char **argv) { - const char outfile[] = "src/ecmult_gen_static_prec_table.h"; + const char outfile[] = "src/precomputed_ecmult_gen.c"; FILE* fp; int bits; @@ -28,21 +28,20 @@ int main(int argc, char **argv) { return -1; } - fprintf(fp, "/* This file was automatically generated by gen_ecmult_gen_static_prec_table. */\n"); + fprintf(fp, "/* This file was automatically generated by precompute_ecmult_gen. */\n"); fprintf(fp, "/* See ecmult_gen_impl.h for details about the contents of this file. */\n"); - fprintf(fp, "#ifndef SECP256K1_ECMULT_GEN_STATIC_PREC_TABLE_H\n"); - fprintf(fp, "#define SECP256K1_ECMULT_GEN_STATIC_PREC_TABLE_H\n"); - + fprintf(fp, "#if defined HAVE_CONFIG_H\n"); + fprintf(fp, "# include \"libsecp256k1-config.h\"\n"); + fprintf(fp, "#endif\n"); + fprintf(fp, "#include \"../include/secp256k1.h\"\n"); fprintf(fp, "#include \"group.h\"\n"); - - fprintf(fp, "#define S(a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p) " - "SECP256K1_GE_STORAGE_CONST(0x##a##u,0x##b##u,0x##c##u,0x##d##u,0x##e##u,0x##f##u,0x##g##u," - "0x##h##u,0x##i##u,0x##j##u,0x##k##u,0x##l##u,0x##m##u,0x##n##u,0x##o##u,0x##p##u)\n"); - + fprintf(fp, "#include \"ecmult_gen.h\"\n"); + fprintf(fp, "#include \"precomputed_ecmult_gen.h\"\n"); fprintf(fp, "#ifdef EXHAUSTIVE_TEST_ORDER\n"); - fprintf(fp, "static secp256k1_ge_storage secp256k1_ecmult_gen_prec_table[ECMULT_GEN_PREC_N(ECMULT_GEN_PREC_BITS)][ECMULT_GEN_PREC_G(ECMULT_GEN_PREC_BITS)];\n"); - fprintf(fp, "#else\n"); - fprintf(fp, "static const secp256k1_ge_storage secp256k1_ecmult_gen_prec_table[ECMULT_GEN_PREC_N(ECMULT_GEN_PREC_BITS)][ECMULT_GEN_PREC_G(ECMULT_GEN_PREC_BITS)] = {\n"); + fprintf(fp, "# error Cannot compile precomputed_ecmult_gen.c in exhaustive test mode\n"); + fprintf(fp, "#endif /* EXHAUSTIVE_TEST_ORDER */\n"); + fprintf(fp, "#define S(a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p) SECP256K1_GE_STORAGE_CONST(0x##a##u,0x##b##u,0x##c##u,0x##d##u,0x##e##u,0x##f##u,0x##g##u,0x##h##u,0x##i##u,0x##j##u,0x##k##u,0x##l##u,0x##m##u,0x##n##u,0x##o##u,0x##p##u)\n"); + fprintf(fp, "const secp256k1_ge_storage secp256k1_ecmult_gen_prec_table[ECMULT_GEN_PREC_N(ECMULT_GEN_PREC_BITS)][ECMULT_GEN_PREC_G(ECMULT_GEN_PREC_BITS)] = {\n"); for (bits = 2; bits <= 8; bits *= 2) { int g = ECMULT_GEN_PREC_G(bits); @@ -50,7 +49,7 @@ int main(int argc, char **argv) { int inner, outer; secp256k1_ge_storage* table = checked_malloc(&default_error_callback, n * g * sizeof(secp256k1_ge_storage)); - secp256k1_ecmult_gen_create_prec_table(table, &secp256k1_ge_const_g, bits); + secp256k1_ecmult_gen_compute_table(table, &secp256k1_ge_const_g, bits); fprintf(fp, "#if ECMULT_GEN_PREC_BITS == %d\n", bits); for(outer = 0; outer != n; outer++) { @@ -74,9 +73,7 @@ int main(int argc, char **argv) { } fprintf(fp, "};\n"); - fprintf(fp, "#endif /* EXHAUSTIVE_TEST_ORDER */\n"); - fprintf(fp, "#undef SC\n"); - fprintf(fp, "#endif /* SECP256K1_ECMULT_GEN_STATIC_PREC_TABLE_H */\n"); + fprintf(fp, "#undef S\n"); fclose(fp); return 0; diff --git a/src/secp256k1/src/ecmult_static_pre_g.h b/src/secp256k1/src/precomputed_ecmult.c index 9072fb2688..3e67f37b74 100644 --- a/src/secp256k1/src/ecmult_static_pre_g.h +++ b/src/secp256k1/src/precomputed_ecmult.c @@ -1,187 +1,38 @@ -/* This file was automatically generated by gen_ecmult_static_pre_g. */ +/* This file was automatically generated by precompute_ecmult. */ /* This file contains an array secp256k1_pre_g with odd multiples of the base point G and * an array secp256k1_pre_g_128 with odd multiples of 2^128*G for accelerating the computation of a*P + b*G. */ -#ifndef SECP256K1_ECMULT_STATIC_PRE_G_H -#define SECP256K1_ECMULT_STATIC_PRE_G_H -#include "group.h" -#ifdef S - #error macro identifier S already in use. +#if defined HAVE_CONFIG_H +# include "libsecp256k1-config.h" #endif +#include "../include/secp256k1.h" +#include "group.h" +#include "ecmult.h" +#include "precomputed_ecmult.h" #define S(a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p) SECP256K1_GE_STORAGE_CONST(0x##a##u,0x##b##u,0x##c##u,0x##d##u,0x##e##u,0x##f##u,0x##g##u,0x##h##u,0x##i##u,0x##j##u,0x##k##u,0x##l##u,0x##m##u,0x##n##u,0x##o##u,0x##p##u) -#if ECMULT_TABLE_SIZE(ECMULT_WINDOW_SIZE) > 8192 - #error configuration mismatch, invalid ECMULT_WINDOW_SIZE. Try deleting ecmult_static_pre_g.h before the build. +#if ECMULT_WINDOW_SIZE > 15 + #error configuration mismatch, invalid ECMULT_WINDOW_SIZE. Try deleting precomputed_ecmult.c before the build. #endif -#if defined(EXHAUSTIVE_TEST_ORDER) -#if EXHAUSTIVE_TEST_ORDER == 13 -#define WINDOW_G 4 -static const secp256k1_ge_storage secp256k1_pre_g[ECMULT_TABLE_SIZE(WINDOW_G)] = { - S(c3459c3d,35326167,cd86cce8,7a2417f,5b8bd567,de8538ee,d507b0c,d128f5bb,8e467fec,cd30000a,6cc1184e,25d382c2,a2f4494e,2fbe9abc,8b64abac,d005fb24) -,S(ae64a1bd,38872f22,f637b457,125cc859,e4c7a31b,cf553cf5,b96e7096,cc61cc10,8e467fec,cd30000a,6cc1184e,25d382c2,a2f4494e,2fbe9abc,8b64abac,d005fb24) -,S(851695d4,9a83f8ef,919bb861,53cbcb16,630fb68a,ed0a766a,3ec693d6,8e6afa40,3c915051,a5fb60b4,fec49de6,e4385101,59f30035,b6502bc0,f5edba86,9753a96c) -,S(0,0,0,0,0,0,0,1,c36eafae,5a049f4b,13b6219,1bc7aefe,a60cffca,49afd43f,a124578,68ac52c3) -}; -static const secp256k1_ge_storage secp256k1_pre_g_128[ECMULT_TABLE_SIZE(WINDOW_G)] = { - S(8e55c205,92466f75,3c417ec0,e600f626,bfac877c,52258a1c,3941145a,62753693,8e467fec,cd30000a,6cc1184e,25d382c2,a2f4494e,2fbe9abc,8b64abac,d005fb24) -,S(c3459c3d,35326167,cd86cce8,7a2417f,5b8bd567,de8538ee,d507b0c,d128f5bb,8e467fec,cd30000a,6cc1184e,25d382c2,a2f4494e,2fbe9abc,8b64abac,d005fb24) -,S(0,0,0,0,0,0,0,1,3c915051,a5fb60b4,fec49de6,e4385101,59f30035,b6502bc0,f5edba86,9753a96c) -,S(7ae96a2b,657c0710,6e64479e,ac3434e9,9cf04975,12f58995,c1396c28,719501ee,c36eafae,5a049f4b,13b6219,1bc7aefe,a60cffca,49afd43f,a124578,68ac52c3) -}; -#elif EXHAUSTIVE_TEST_ORDER == 199 -#define WINDOW_G 8 -static const secp256k1_ge_storage secp256k1_pre_g[ECMULT_TABLE_SIZE(WINDOW_G)] = { - S(226e653f,c8df7744,9bacbf12,7d1dcbf9,87f05b2a,e7edbd28,1f564575,c48dcf18,a13872c2,e933bb17,5d9ffd5b,b5b6e10c,57fe3c00,baaaa15a,e003ec3e,9c269bae) -,S(ff1755e,623c8369,f55edda4,2a5deef0,b32c57f4,80c5884f,d2a2dde1,b1c078c4,640db9f3,9dda2f51,ee3ef3db,775315aa,c06346f1,e31ff76f,83a24bb0,8fc93242) -,S(64dd1439,5d19a544,a7a1e81b,b9d079b3,593e7022,6bcd444e,6dc8197a,1a6dc3e6,2c7f2dce,e421d852,d3bff68e,993c8bb4,c189d3be,bd4fa667,6a599f9f,8c639c50) -,S(199d1eb1,5a28f0aa,7258651b,ff07ff13,1c988fa,dc41dc67,390c3172,54b98016,d670e3f1,6fc2382,46bf323f,127e7c14,576e3a64,5d5c41da,40f22b29,4fca876c) -,S(3d303d12,f8eb67d0,ecaee514,bdd90e57,b58b6d6a,c896a26f,78c06103,52225b04,d282f481,b04bece2,60de6995,b58c4f0d,9c6121e0,d94f45da,f5da7f13,cfefef99) -,S(76bb4d25,c7a005dc,49a5295f,bb92bf1e,5d0dd5f6,609c2008,54f37361,23a6f9f3,e2295c71,21478f52,d4d18aa,72cd82ae,1995d37e,6ef2e05a,4dea6ee2,6371fbe3) -,S(8735fb32,6950ae71,31cd03f7,cc1511d8,65c87e84,748c9824,8ccc576e,5480ed8,39cfeb15,665f371f,535d56c1,317a8d62,29ea2827,943a98f1,cee7f685,86e47eec) -,S(8521a020,a8e7cc5e,d9534d74,d4656581,33869bc,1940548d,6cc35ba4,e3c4323a,c63014ea,99a0c8e0,aca2a93e,ce85729d,d615d7d8,6bc5670e,31180979,791b7d43) -,S(2f63a396,4d14ca73,2167e033,306863d8,f75605c7,4ea6c917,890dd50d,fc20f53b,e2295c71,21478f52,d4d18aa,72cd82ae,1995d37e,6ef2e05a,4dea6ee2,6371fbe3) -,S(c479fee8,e2202b42,1c9346fb,4e59f720,95a70dea,2ade9641,58bad01c,2c37f,6897db9d,5a3d8948,ea97069e,de75661,b1ad74e4,2b1daa4f,533d88ba,67137731) -,S(4d756b2,e61d9806,e02ec830,f1fd44f8,46438689,c528346,15a1addf,ac864bcd,c90704c9,8f72d8be,638a36dc,e8768909,fac7921c,dc54d314,89fbb3e,6da9018a) -,S(baeab1a8,bca6e88,740deeb5,93887afe,295155ba,fa50f307,baf2de95,2841dca6,a88e0162,de086e20,587faa5c,bca8f572,318c17e9,5668819c,13c4d5d9,2d1c75dc) -,S(47bbafcd,f612ca3e,703f7721,623bde14,926fc00d,a6bf9e9,41ddb2d5,587626e3,d160b834,5807531a,327a7269,3627dab6,f9770a12,14eb182d,fa73ad15,41b57ff0) -,S(f9d5cda0,a41aea0e,f0afc533,e1b50808,3acc6043,e04e55bb,77421ab,1cebb8ec,f8a4cc8e,538e1b7f,627c6713,7c896eb1,c5548159,f434695a,ea394c63,592d41cf) -,S(26e2e75b,aeb04a72,8b893f67,d27ebfc6,b8984a88,6fbd8eee,5719107d,5a413f0f,61b51308,ac8643f0,7c18eda7,ef9a4196,aa66221,1b809acc,2b2cb538,23010ef8) -,S(318ef624,4d411c97,913a7bb3,8f07b960,e6c57afa,f1edc5bf,e708da8a,954f68ac,37d97203,d25d9c14,53827e27,9474afb2,92943df0,8ba1108a,43644c86,ee6c87e4) -,S(eac612b0,84cb452a,35d49c60,c0d0c0bd,359eb909,68a80186,92a028f,ece4c331,23575727,9da3d12b,c0004d12,e06a9cda,8405c91e,b8118260,2793e62f,16c2a29e) -,S(9f5ed380,fa5188a1,cb1c84df,cb7e2aad,939cdfde,581d765b,95c1c25,43016afb,423923ad,75aa7eb1,259434a7,1d58c469,f6268a38,5cd8fb7c,4be64d63,fad9bcea) -,S(ee534645,42ff07e2,8cf783ee,1da96e00,b81f531,9b9a4885,27f92302,acd12cac,c90704c9,8f72d8be,638a36dc,e8768909,fac7921c,dc54d314,89fbb3e,6da9018a) -,S(1130e02f,29480660,b2cb4f68,81da3f2c,1bdaf64,8df3b6ce,1a333738,29e4165e,ef2b477b,a9c921df,5ff65b6e,7c6f96a8,627743ff,a53e3858,cd9d7673,f7435ac4) -,S(e8d3f45a,b5be455f,dc3c2270,34100ca2,b8d203e1,35a9ed3d,5c8b703c,ef4d9b48,2d7d0b7e,4fb4131d,9f21966a,4a73b0f2,639ede1f,26b0ba25,a2580eb,30100c96) -,S(eb5c4120,4917d176,6fea883b,e680dfd9,233d9654,97fac48b,76f5d0d4,7f6dbf73,97682462,a5c276b7,1568f961,f218a99e,4e528b1b,d4e255b0,acc27744,98ec84fe) -,S(4097892c,5478baea,62e8da20,6d17fcd8,b8eed45f,24fb7648,fe742508,8da60a4e,10d4b884,5636de20,a009a491,83906957,9d88bc00,5ac1c7a7,3262898b,8bca16b) -,S(9963e528,754cbba8,29622664,1aec11b0,3fba2739,40f61cbd,5d4c0478,5429f40,d670e3f1,6fc2382,46bf323f,127e7c14,576e3a64,5d5c41da,40f22b29,4fca876c) -,S(853cffd4,4cb0a29e,390ad886,455566cc,2776ab74,55210e57,9bf1fcc0,c8d767ed,621fdd8d,dd929f3f,681a6f83,bf8061f8,2b4396a6,5df86be6,ec656a86,b1b730b0) -,S(7fe309bb,e3e8edd3,3f5f94b,4247256e,41e62e69,fd9f2063,d878c6d6,3d85675f,23575727,9da3d12b,c0004d12,e06a9cda,8405c91e,b8118260,2793e62f,16c2a29e) -,S(851695d4,9a83f8ef,919bb861,53cbcb16,630fb68a,ed0a766a,3ec693d6,8e6afa40,cb47a6a4,91313828,f036dbd7,64173a40,473d3969,5cc47758,511d0a81,badb02f1) -,S(f31a7dfe,4d2e4a98,aaf3dd70,25441c7a,2da056b0,db6795e8,3a39d76f,b316bd92,17e98bbd,6e144db,959b7aea,cd766958,7be26db5,445ea19d,dab8bc90,1f527cb4) -,S(2266b5e,d5cef55,81e36315,c5fc5c53,a1c73b12,c24c8de7,9afef2a0,7a50212c,7a60fc27,e4ab0ba6,ea71c036,52b83c5c,5ab5dac8,ea43dec2,84719f8c,6d7c3c81) -,S(e0089ec9,4b25ac6,ff44b5da,2dfedb37,9fdaedf1,6bc8cecd,b1a95fa0,32431c73,17e98bbd,6e144db,959b7aea,cd766958,7be26db5,445ea19d,dab8bc90,1f527cb4) -,S(896c7bc,5cfeaecc,37b111ea,2deee270,c7cb24eb,6eb1cb80,a84e87d3,89d997c1,66446d0d,b01a679b,59afb34a,2bbf8cfe,f0a44870,f0e074ce,78a51a36,9bbbf33c) -,S(709c8403,ed2fd0f9,5fac44f0,dfa96ec2,e7b357e4,4ebb3dcf,6f688f20,39d3d67d,d160b834,5807531a,327a7269,3627dab6,f9770a12,14eb182d,fa73ad15,41b57ff0) -,S(142f854d,ba0b87a2,9826f018,86ab51ff,b9f19214,c23c07c0,f008a375,a1effdb6,74bafd32,bdf2bdfd,c65a52c3,5ef4f4d,9332f7b2,a0512a9f,f821f858,484bb9c) -,S(66abbf18,f58dd1c9,4b6072c,eb182abe,c31d3af9,6d91060b,d233f462,5716f289,d380d231,1bde27ad,2c400971,66c3744b,3e762c41,42b05998,95a6605f,739c5fdf) -,S(7a1b727f,c05c6de2,79b2e027,108fae3e,87ca9249,eb44207f,286e1cbe,4728027,9de02272,226d60c0,97e5907c,407f9e07,d4bc6959,a2079419,139a9578,4e48cb7f) -,S(3d98fe85,5d42f661,e35d8ce3,cfee5f17,57bc3de9,ecec02fa,11a2d90,378ac0e8,bdc6dc52,8a55814e,da6bcb58,e2a73b96,9d975c7,a3270483,b419b29b,5263f45) -,S(34f8450b,18bffd3f,cbadcd08,5da5411b,ba2ef195,55753768,bf777307,3a2d4cb4,5771fe9d,21f791df,a78055a3,43570a8d,ce73e816,a9977e63,ec3b2a25,d2e38653) -,S(76dac07e,bc240289,791e6a9a,56abdc7,bdf2fac1,e7a0100b,5a0ee09e,3455cc33,8b4502cd,420d4202,39a5ad3c,fa10b0b2,6ccd084d,5faed560,7de07a6,fb7b4093) -,S(929effed,d3b91ccb,a985ab23,bef45c16,6d5ffaa3,22562529,b84d051c,f94044a1,640db9f3,9dda2f51,ee3ef3db,775315aa,c06346f1,e31ff76f,83a24bb0,8fc93242) -,S(1ea4ff3b,5a0a78a9,fda9d73a,ea5f0629,886b083a,6d3beea4,d6bc96e1,b50da944,9edf1300,7f2b7fb2,92301e8c,77ddb63e,5f2d23c4,315118c1,8b1b6273,86bba11a) -,S(86fef7a7,b4d3b079,2c83a223,d1c929d6,6ca15202,f4b956a1,cf7ed2cb,83dba7c4,61b51308,ac8643f0,7c18eda7,ef9a4196,aa66221,1b809acc,2b2cb538,23010ef8) -,S(c979de1c,1318f60f,a9929ff3,a6621b9,d15226e8,2883556e,36a5f77b,28e3b585,d7bd974a,b40a1bd3,a35a1ee9,16d32768,48e7bcfc,be70851d,8ba758b9,996d2099) -,S(2cdce338,ae1f5aa0,55c76cb5,acbd084e,3284bb5d,b8cf9b4a,141cc8ee,1aa61e59,17e98bbd,6e144db,959b7aea,cd766958,7be26db5,445ea19d,dab8bc90,1f527cb4) -,S(80c5672b,3f773975,b5829101,6fa9de9b,33863263,8a978db,934f04c1,3a909b85,55ebf47c,6835f232,5ea232b3,8af64478,65cbdce1,2f2c5a79,abc445fd,518ab677) -,S(521e20fc,9c7c0514,47f31e74,5bb81662,dac66374,9b891a6f,d9681cb6,21e3155c,61b51308,ac8643f0,7c18eda7,ef9a4196,aa66221,1b809acc,2b2cb538,23010ef8) -,S(b084fee5,5da62ee0,ebdebf3b,245e0d4e,def46aac,bfdc9f43,261d3e2b,eed946a0,80f3b79c,ba5f8457,3f2aa473,1d10c1f,9b6543e1,b8cec5e0,e89cd6df,868cc2aa) -,S(d8a3fd95,b7ed8e8a,674da307,94911393,303917c2,60944d1f,76261274,fba45757,5ec78d3d,16cc44e8,a26002a4,4a491ef3,a801c3ff,45555ea5,1ffc13c0,63d96081) -,S(c2d4b350,75ee5eb,3e2c8d3c,f33649dc,dd4f4c29,728bbdae,3f3ff02d,f3dc7415,6120ecff,80d4804d,6dcfe173,882249c1,a0d2dc3b,ceaee73e,74e49d8b,79445b15) -,S(9fa3c7bd,6b8544f2,df48309,596ea267,8f8537e1,dc406ec7,cf2c67d1,ae5f1e8a,c8268dfc,2da263eb,ac7d81d8,6b8b504d,6d6bc20f,745eef75,bc9bb378,1193744b) -,S(0,0,0,0,0,0,0,1,34b8595b,6ecec7d7,fc92428,9be8c5bf,b8c2c696,a33b88a7,aee2f57d,4524f93e) -,S(969360d2,54e2dba9,e79d0789,c339250d,c438330a,a66addf9,f27e2ee8,fc9036ac,99bb92f2,4fe59864,a6504cb5,d4407301,f5bb78f,f1f8b31,875ae5c8,644408f3) -,S(318f2c1f,3dc109ad,ed9d8958,e2e6eb8c,e26eec32,f2725ad1,68e116c7,497e7044,75b3371,ac71e480,9d8398ec,8376914e,3aab7ea6,bcb96a5,15c6b39b,a6d2ba60) -,S(bcea96e9,4ff1b11e,6e8a09d6,ec594b00,597e2b0a,37b0e581,50a58d59,33220419,7f0c4863,45a07ba8,c0d55b8c,fe2ef3e0,649abc1e,47313a1f,1763291f,79733985) -,S(4ed9d2a,7f32fa30,fd059de5,ee512073,47d68d12,b77df5b8,6a83a814,3fcdd5c0,5ec78d3d,16cc44e8,a26002a4,4a491ef3,a801c3ff,45555ea5,1ffc13c0,63d96081) -,S(875c93a1,8ac5a86e,9e85e4aa,3c5ad0ac,3d697192,6384b05b,568604b3,8fee8fb6,3738dc95,97add4e5,c0a39af1,b98ccba2,7bb43333,e35ab481,54c52779,9191144b) -,S(47a7cc2e,1cbd64c8,301443ed,be1ab328,85dce80e,a6d8c847,4eb9be09,6db5fecf,d160b834,5807531a,327a7269,3627dab6,f9770a12,14eb182d,fa73ad15,41b57ff0) -,S(3e8a6608,17f28243,f363d67e,b874856f,1fc9810a,9da6c3f3,a72dd23f,ec45513e,8a8d4c8f,37d2572,6ca66369,59c9eca3,82bd076e,867092a0,b428ad99,2c54a63b) -,S(9556e393,974bcd02,c6356a53,fce819d4,887b188c,99b8de16,1e5d3697,d595cdce,23575727,9da3d12b,c0004d12,e06a9cda,8405c91e,b8118260,2793e62f,16c2a29e) -,S(5245e215,52c6f785,2cccef32,e9f54774,cfb6bb9b,c363ae72,71174df4,6d77fdcc,ca78f1f0,894a6670,19609169,78c251dc,64f969e,c25be4b4,202508d3,22d07f3b) -,S(5d6f8ab3,ca0a5fca,611b7738,16adb4f8,df73ad68,5ce45286,75101d00,54ff3eca,640db9f3,9dda2f51,ee3ef3db,775315aa,c06346f1,e31ff76f,83a24bb0,8fc93242) -,S(a4465b18,b7f702e2,a8165184,834ce490,68fb0a51,6658734b,6229eda0,605911a2,c8c7236a,68522b1a,3f5c650e,4673345d,844bcccc,1ca54b7e,ab3ad885,6e6ee7e4) -,S(d49b0640,1e240c43,21b2b173,3b640c6a,e2c4b389,2d3f4f73,8faac78b,9995cf2e,75b3371,ac71e480,9d8398ec,8376914e,3aab7ea6,bcb96a5,15c6b39b,a6d2ba60) -,S(599db82f,8229ad47,7a662726,d2d351a6,f76cefd7,8fc376b1,ef6e344e,4e4dae3a,284268b5,4bf5e42c,5ca5e116,e92cd897,b7184303,418f7ae2,7458a745,6692db96) -,S(a0627644,ebc6a423,1f3e113b,eedbf9d8,21d9374a,af2e7c55,14ee7395,aa9992b7,859f03d8,1b54f459,158e3fc9,ad47c3a3,a54a2537,15bc213d,7b8e6072,9283bfae) -}; -static const secp256k1_ge_storage secp256k1_pre_g_128[ECMULT_TABLE_SIZE(WINDOW_G)] = { - S(ec823227,1d0e875a,8e1cf7e6,f9a35a15,749be650,1cc885c4,f383060b,46bd3d33,aa140b83,97ca0dcd,a15dcd4c,7509bb87,9a34231e,d0d3a586,543bba01,ae7545b8) -,S(66abbf18,f58dd1c9,4b6072c,eb182abe,c31d3af9,6d91060b,d233f462,5716f289,d380d231,1bde27ad,2c400971,66c3744b,3e762c41,42b05998,95a6605f,739c5fdf) -,S(4d756b2,e61d9806,e02ec830,f1fd44f8,46438689,c528346,15a1addf,ac864bcd,36f8fb36,708d2741,9c75c923,178976f6,5386de3,23ab2ceb,f76044c0,9256faa5) -,S(875c93a1,8ac5a86e,9e85e4aa,3c5ad0ac,3d697192,6384b05b,568604b3,8fee8fb6,c8c7236a,68522b1a,3f5c650e,4673345d,844bcccc,1ca54b7e,ab3ad885,6e6ee7e4) -,S(f3a864ac,edc7852f,f4dfae93,5f8588a6,96ff17bf,7233134e,6704ceb,16f3b74c,39cfeb15,665f371f,535d56c1,317a8d62,29ea2827,943a98f1,cee7f685,86e47eec) -,S(3e8a6608,17f28243,f363d67e,b874856f,1fc9810a,9da6c3f3,a72dd23f,ec45513e,8a8d4c8f,37d2572,6ca66369,59c9eca3,82bd076e,867092a0,b428ad99,2c54a63b) -,S(47bbafcd,f612ca3e,703f7721,623bde14,926fc00d,a6bf9e9,41ddb2d5,587626e3,d160b834,5807531a,327a7269,3627dab6,f9770a12,14eb182d,fa73ad15,41b57ff0) -,S(709c8403,ed2fd0f9,5fac44f0,dfa96ec2,e7b357e4,4ebb3dcf,6f688f20,39d3d67d,2e9f47cb,a7f8ace5,cd858d96,c9d82549,688f5ed,eb14e7d2,58c52e9,be4a7c3f) -,S(66fd0ada,7159c2df,fdd925e7,2eb70772,743e39e3,4c10d5cf,f58eac4e,a30e4157,8a8d4c8f,37d2572,6ca66369,59c9eca3,82bd076e,867092a0,b428ad99,2c54a63b) -,S(60d5d771,4e1e7589,e0b1e68c,ed7f881,73fca809,eae35685,65334942,79962dc2,99bb92f2,4fe59864,a6504cb5,d4407301,f5bb78f,f1f8b31,875ae5c8,644408f3) -,S(3d98fe85,5d42f661,e35d8ce3,cfee5f17,57bc3de9,ecec02fa,11a2d90,378ac0e8,bdc6dc52,8a55814e,da6bcb58,e2a73b96,9d975c7,a3270483,b419b29b,5263f45) -,S(2f63a396,4d14ca73,2167e033,306863d8,f75605c7,4ea6c917,890dd50d,fc20f53b,1dd6a38e,deb870ad,f2b2e755,8d327d51,e66a2c81,910d1fa5,b215911c,9c8e004c) -,S(bcea96e9,4ff1b11e,6e8a09d6,ec594b00,597e2b0a,37b0e581,50a58d59,33220419,80f3b79c,ba5f8457,3f2aa473,1d10c1f,9b6543e1,b8cec5e0,e89cd6df,868cc2aa) -,S(ae3796a4,823f3eb4,ea4bd677,110dc3fb,45537c3c,4d10d2e8,e758a3be,4875db83,ef2b477b,a9c921df,5ff65b6e,7c6f96a8,627743ff,a53e3858,cd9d7673,f7435ac4) -,S(5245e215,52c6f785,2cccef32,e9f54774,cfb6bb9b,c363ae72,71174df4,6d77fdcc,ca78f1f0,894a6670,19609169,78c251dc,64f969e,c25be4b4,202508d3,22d07f3b) -,S(26e2e75b,aeb04a72,8b893f67,d27ebfc6,b8984a88,6fbd8eee,5719107d,5a413f0f,61b51308,ac8643f0,7c18eda7,ef9a4196,aa66221,1b809acc,2b2cb538,23010ef8) -,S(e0089ec9,4b25ac6,ff44b5da,2dfedb37,9fdaedf1,6bc8cecd,b1a95fa0,32431c73,e8167442,f91ebb24,6a648515,328996a7,841d924a,bba15e62,2547436e,e0ad7f7b) -,S(f8f16642,266d8dc6,abfdbbc6,f6c47711,a192c051,7f00a633,b78076a7,a7884e72,4f6d0e5,148ce7e1,772f84c,b08903b8,71d2beb5,d0934488,8ab1c75d,42232790) -,S(23082df9,a86b80fc,5185ee3c,6493763b,14a6e237,baf686aa,f589b649,8573d04c,bdc6dc52,8a55814e,da6bcb58,e2a73b96,9d975c7,a3270483,b419b29b,5263f45) -,S(76dac07e,bc240289,791e6a9a,56abdc7,bdf2fac1,e7a0100b,5a0ee09e,3455cc33,8b4502cd,420d4202,39a5ad3c,fa10b0b2,6ccd084d,5faed560,7de07a6,fb7b4093) -,S(8735fb32,6950ae71,31cd03f7,cc1511d8,65c87e84,748c9824,8ccc576e,5480ed8,c63014ea,99a0c8e0,aca2a93e,ce85729d,d615d7d8,6bc5670e,31180979,791b7d43) -,S(969360d2,54e2dba9,e79d0789,c339250d,c438330a,a66addf9,f27e2ee8,fc9036ac,66446d0d,b01a679b,59afb34a,2bbf8cfe,f0a44870,f0e074ce,78a51a36,9bbbf33c) -,S(74f5ba33,89d075d3,eebaa54d,73e9f038,881b7329,5623e833,b5e87beb,29ba3246,74bafd32,bdf2bdfd,c65a52c3,5ef4f4d,9332f7b2,a0512a9f,f821f858,484bb9c) -,S(a4465b18,b7f702e2,a8165184,834ce490,68fb0a51,6658734b,6229eda0,605911a2,c8c7236a,68522b1a,3f5c650e,4673345d,844bcccc,1ca54b7e,ab3ad885,6e6ee7e4) -,S(eac612b0,84cb452a,35d49c60,c0d0c0bd,359eb909,68a80186,92a028f,ece4c331,23575727,9da3d12b,c0004d12,e06a9cda,8405c91e,b8118260,2793e62f,16c2a29e) -,S(f31a7dfe,4d2e4a98,aaf3dd70,25441c7a,2da056b0,db6795e8,3a39d76f,b316bd92,e8167442,f91ebb24,6a648515,328996a7,841d924a,bba15e62,2547436e,e0ad7f7b) -,S(470ce05d,53f77ff0,c5ab8770,a4c11a3e,23f7d4fe,f5c0ab36,1b940a8a,6b8cf7b9,57d6adb1,5f0b1daa,1fce91d7,ccf75d45,9480b056,6f81ff8a,85612173,1cd8f2b8) -,S(d9fbce92,515652cf,3714f87b,e16e505,91a28eb4,1bf7053,2ab42ebd,be900212,2d7d0b7e,4fb4131d,9f21966a,4a73b0f2,639ede1f,26b0ba25,a2580eb,30100c96) -,S(1ea4ff3b,5a0a78a9,fda9d73a,ea5f0629,886b083a,6d3beea4,d6bc96e1,b50da944,9edf1300,7f2b7fb2,92301e8c,77ddb63e,5f2d23c4,315118c1,8b1b6273,86bba11a) -,S(3d303d12,f8eb67d0,ecaee514,bdd90e57,b58b6d6a,c896a26f,78c06103,52225b04,2d7d0b7e,4fb4131d,9f21966a,4a73b0f2,639ede1f,26b0ba25,a2580eb,30100c96) -,S(9fa3c7bd,6b8544f2,df48309,596ea267,8f8537e1,dc406ec7,cf2c67d1,ae5f1e8a,37d97203,d25d9c14,53827e27,9474afb2,92943df0,8ba1108a,43644c86,ee6c87e4) -,S(92906a31,52682000,a59736ed,ef48a7b0,c78d6a49,8727b3b,893d3478,de04ada5,80f3b79c,ba5f8457,3f2aa473,1d10c1f,9b6543e1,b8cec5e0,e89cd6df,868cc2aa) -,S(599db82f,8229ad47,7a662726,d2d351a6,f76cefd7,8fc376b1,ef6e344e,4e4dae3a,284268b5,4bf5e42c,5ca5e116,e92cd897,b7184303,418f7ae2,7458a745,6692db96) -,S(ee534645,42ff07e2,8cf783ee,1da96e00,b81f531,9b9a4885,27f92302,acd12cac,c90704c9,8f72d8be,638a36dc,e8768909,fac7921c,dc54d314,89fbb3e,6da9018a) -,S(7fe309bb,e3e8edd3,3f5f94b,4247256e,41e62e69,fd9f2063,d878c6d6,3d85675f,dca8a8d8,625c2ed4,3fffb2ed,1f956325,7bfa36e1,47ee7d9f,d86c19cf,e93d5991) -,S(64eaaad1,9c7e9d01,5cc997ff,c2f23022,c59805d,48a48e1,970b8847,10211659,fb092f1a,eb73181e,f88d07b3,4f76fc47,8e2d414a,2f6cbb77,754e38a1,bddcd49f) -,S(59e10f43,eb4b2fb0,94f2f66d,1404dd08,ab9c2442,50bd16e0,21feb78f,e0380d01,e2295c71,21478f52,d4d18aa,72cd82ae,1995d37e,6ef2e05a,4dea6ee2,6371fbe3) -,S(c979de1c,1318f60f,a9929ff3,a6621b9,d15226e8,2883556e,36a5f77b,28e3b585,d7bd974a,b40a1bd3,a35a1ee9,16d32768,48e7bcfc,be70851d,8ba758b9,996d2099) -,S(64dd1439,5d19a544,a7a1e81b,b9d079b3,593e7022,6bcd444e,6dc8197a,1a6dc3e6,d380d231,1bde27ad,2c400971,66c3744b,3e762c41,42b05998,95a6605f,739c5fdf) -,S(d8a3fd95,b7ed8e8a,674da307,94911393,303917c2,60944d1f,76261274,fba45757,a13872c2,e933bb17,5d9ffd5b,b5b6e10c,57fe3c00,baaaa15a,e003ec3e,9c269bae) -,S(acee4285,57fc32a1,d3d14199,30cadaf0,613a630a,7a7e8c14,d08d05df,19143c3a,ca78f1f0,894a6670,19609169,78c251dc,64f969e,c25be4b4,202508d3,22d07f3b) -,S(5d771e5d,6dc6c87,5ede8bae,4b27a9d4,3c5f8da2,8e84f5c3,501299c8,db16484c,859f03d8,1b54f459,158e3fc9,ad47c3a3,a54a2537,15bc213d,7b8e6072,9283bfae) -,S(e8d3f45a,b5be455f,dc3c2270,34100ca2,b8d203e1,35a9ed3d,5c8b703c,ef4d9b48,2d7d0b7e,4fb4131d,9f21966a,4a73b0f2,639ede1f,26b0ba25,a2580eb,30100c96) -,S(9963e528,754cbba8,29622664,1aec11b0,3fba2739,40f61cbd,5d4c0478,5429f40,298f1c0e,f903dc7d,b940cdc0,ed8183eb,a891c59b,a2a3be25,bf0dd4d5,b03574c3) -,S(cbdb65,553cd5d8,ff61cf33,e53fdd9a,cf0ee159,c21dc578,be5bac2b,7973c229,ca78f1f0,894a6670,19609169,78c251dc,64f969e,c25be4b4,202508d3,22d07f3b) -,S(5029bff6,d4c80347,738230c8,cb252906,471b5bc1,3d26a533,304f5f0d,808f756c,97682462,a5c276b7,1568f961,f218a99e,4e528b1b,d4e255b0,acc27744,98ec84fe) -,S(80c5672b,3f773975,b5829101,6fa9de9b,33863263,8a978db,934f04c1,3a909b85,55ebf47c,6835f232,5ea232b3,8af64478,65cbdce1,2f2c5a79,abc445fd,518ab677) -,S(226e653f,c8df7744,9bacbf12,7d1dcbf9,87f05b2a,e7edbd28,1f564575,c48dcf18,5ec78d3d,16cc44e8,a26002a4,4a491ef3,a801c3ff,45555ea5,1ffc13c0,63d96081) -,S(521e20fc,9c7c0514,47f31e74,5bb81662,dac66374,9b891a6f,d9681cb6,21e3155c,9e4aecf7,5379bc0f,83e71258,1065be69,f5599dde,e47f6533,d4d34ac6,dcfeed37) -,S(5472b5d0,4ba1ca80,236183b1,46e596e5,4b20b83b,248988c8,bbe8460e,3666f262,a829524e,a0f4e255,e0316e28,3308a2ba,6b7f4fa9,907e0075,7a9ede8b,e3270977) -,S(2ecd421e,47399e76,60d10143,1789a437,89b54d23,31d1cb78,49cabda2,bc5174f9,c8268dfc,2da263eb,ac7d81d8,6b8b504d,6d6bc20f,745eef75,bc9bb378,1193744b) -,S(4097892c,5478baea,62e8da20,6d17fcd8,b8eed45f,24fb7648,fe742508,8da60a4e,10d4b884,5636de20,a009a491,83906957,9d88bc00,5ac1c7a7,3262898b,8bca16b) -,S(eb5c4120,4917d176,6fea883b,e680dfd9,233d9654,97fac48b,76f5d0d4,7f6dbf73,6897db9d,5a3d8948,ea97069e,de75661,b1ad74e4,2b1daa4f,533d88ba,67137731) -,S(92b866ad,a37a3f2f,bc607717,96b2c74f,57dde74c,da8e015f,792df531,7eb21fa6,55ebf47c,6835f232,5ea232b3,8af64478,65cbdce1,2f2c5a79,abc445fd,518ab677) -,S(a78dab,f2f2ef7f,4d424752,aa1aeaf5,50bec241,bf9ad129,3b9fe680,32b6141b,9de02272,226d60c0,97e5907c,407f9e07,d4bc6959,a2079419,139a9578,4e48cb7f) -,S(b084fee5,5da62ee0,ebdebf3b,245e0d4e,def46aac,bfdc9f43,261d3e2b,eed946a0,80f3b79c,ba5f8457,3f2aa473,1d10c1f,9b6543e1,b8cec5e0,e89cd6df,868cc2aa) -,S(ff1755e,623c8369,f55edda4,2a5deef0,b32c57f4,80c5884f,d2a2dde1,b1c078c4,640db9f3,9dda2f51,ee3ef3db,775315aa,c06346f1,e31ff76f,83a24bb0,8fc93242) -,S(2cdce338,ae1f5aa0,55c76cb5,acbd084e,3284bb5d,b8cf9b4a,141cc8ee,1aa61e59,e8167442,f91ebb24,6a648515,328996a7,841d924a,bba15e62,2547436e,e0ad7f7b) -,S(3faf6744,2aebb675,c6b75911,e3d34f4c,a8f5b669,a4c438f2,2c163659,3e36c1ea,fc0b579e,cbaf2c74,7b5c792d,7b66abf2,dfb1a44d,2ed6a417,7e1fc6bd,a989b623) -,S(34772cad,ad5888f2,53a810b7,5b175b8d,e3a454e4,26a1b5a5,c003f222,8e7b45c0,d380d231,1bde27ad,2c400971,66c3744b,3e762c41,42b05998,95a6605f,739c5fdf) -,S(853cffd4,4cb0a29e,390ad886,455566cc,2776ab74,55210e57,9bf1fcc0,c8d767ed,621fdd8d,dd929f3f,681a6f83,bf8061f8,2b4396a6,5df86be6,ec656a86,b1b730b0) -,S(1130e02f,29480660,b2cb4f68,81da3f2c,1bdaf64,8df3b6ce,1a333738,29e4165e,10d4b884,5636de20,a009a491,83906957,9d88bc00,5ac1c7a7,3262898b,8bca16b) -,S(a0627644,ebc6a423,1f3e113b,eedbf9d8,21d9374a,af2e7c55,14ee7395,aa9992b7,7a60fc27,e4ab0ba6,ea71c036,52b83c5c,5ab5dac8,ea43dec2,84719f8c,6d7c3c81) -,S(1e864d74,9e96a16a,c4299b88,226aaff9,9a45ab9c,203853ac,ea0378ef,5715ded6,6120ecff,80d4804d,6dcfe173,882249c1,a0d2dc3b,ceaee73e,74e49d8b,79445b15) -}; -#else - #error No known generator for the specified exhaustive test group order. -#endif -#else /* !defined(EXHAUSTIVE_TEST_ORDER) */ +#ifdef EXHAUSTIVE_TEST_ORDER +# error Cannot compile precomputed_ecmult.c in exhaustive test mode +#endif /* EXHAUSTIVE_TEST_ORDER */ #define WINDOW_G ECMULT_WINDOW_SIZE -static const secp256k1_ge_storage secp256k1_pre_g[ECMULT_TABLE_SIZE(WINDOW_G)] = { +const secp256k1_ge_storage secp256k1_pre_g[ECMULT_TABLE_SIZE(WINDOW_G)] = { S(79be667e,f9dcbbac,55a06295,ce870b07,29bfcdb,2dce28d9,59f2815b,16f81798,483ada77,26a3c465,5da4fbfc,e1108a8,fd17b448,a6855419,9c47d08f,fb10d4b8) -#if ECMULT_TABLE_SIZE(WINDOW_G) > 1 +#if WINDOW_G > 2 ,S(f9308a01,9258c310,49344f85,f89d5229,b531c845,836f99b0,8601f113,bce036f9,388f7b0f,632de814,fe337e6,2a37f356,6500a999,34c2231b,6cb9fd75,84b8e672) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 2 +#if WINDOW_G > 3 ,S(2f8bde4d,1a072093,55b4a725,a5c5128,e88b84bd,dc619ab7,cba8d569,b240efe4,d8ac2226,36e5e3d6,d4dba9dd,a6c9c426,f788271b,ab0d6840,dca87d3a,a6ac62d6) ,S(5cbdf064,6e5db4ea,a398f365,f2ea7a0e,3d419b7e,330e39c,e92bdded,cac4f9bc,6aebca40,ba255960,a3178d6d,861a54db,a813d0b8,13fde7b5,a5082628,87264da) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 4 +#if WINDOW_G > 4 ,S(acd484e2,f0c7f653,9ad178a,9f559abd,e0979697,4c57e714,c35f110d,fc27ccbe,cc338921,b0a7d9fd,64380971,763b61e9,add888a4,375f8e0f,5cc262a,c64f9c37) ,S(774ae7f8,58a9411e,5ef4246b,70c65aac,5649980b,e5c17891,bbec1789,5da008cb,d984a032,eb6b5e19,243dd56,d7b7b365,372db1e2,dff9d6a8,301d74c9,c953c61b) ,S(f28773c2,d975288b,c7d1d205,c3748651,b075fbc6,610e58cd,deeddf8f,19405aa8,ab0902e,8d880a89,758212eb,65cdaf47,3a1a06da,521fa91f,29b5cb52,db03ed81) ,S(d7924d4f,7d43ea96,5a465ae3,95ff411,31e5946f,3c85f79e,44adbcf8,e27e080e,581e2872,a86c72a6,83842ec2,28cc6def,ea40af2b,d896d3a5,c504dc9f,f6a26b58) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 8 +#if WINDOW_G > 5 ,S(defdea4c,db677750,a420fee8,7eacf21,eb9898ae,79b97687,66e4faa0,4a2d4a34,4211ab06,94635168,e997b0ea,d2a93dae,ced1f4a0,4a95c0f6,cfb199f6,9e56eb77) ,S(2b4ea0a7,97a443d2,93ef5cff,444f4979,f06acfeb,d7e86d27,74756561,38385b6c,85e89bc0,37945d93,b343083b,5a1c8613,1a01f60c,50269763,b570c854,e5c09b7a) ,S(352bbf4a,4cdd1256,4f93fa33,2ce33330,1d9ad402,71f81071,81340aef,25be59d5,321eb407,5348f534,d59c1825,9dda3e1f,4a1b3b2e,71b1039c,67bd3d8b,cf81998c) @@ -191,7 +42,7 @@ static const secp256k1_ge_storage secp256k1_pre_g[ECMULT_TABLE_SIZE(WINDOW_G)] = ,S(c44d12c7,65d812e,8acf28d7,cbb19f90,11ecd9e9,fdf281b0,e6a3b5e8,7d22e7db,2119a460,ce326cdc,76c45926,c982fdac,e106e86,1edf61c5,a039063f,e0e6482) ,S(6a245bf6,dc698504,c89a20cf,ded60853,152b6953,36c28063,b61c65cb,d269e6b4,e022cf42,c2bd4a70,8b3f5126,f16a24ad,8b33ba48,d0423b6e,fd5e6348,100d8a82) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 16 +#if WINDOW_G > 6 ,S(1697ffa6,fd9de627,c077e3d2,fe541084,ce13300b,bec1146,f95ae57f,d0bd6a5,b9c398f1,86806f5d,27561506,e4557433,a2cf1500,9e498ae7,adee9d63,d01b2396) ,S(605bdb01,9981718b,986d0f07,e834cb0d,9deb8360,ffb7f61d,f982345e,f27a7479,2972d2d,e4f8d206,81a78d93,ec96fe23,c26bfae8,4fb14db4,3b01e1e9,56b8c49) ,S(62d14dab,4150bf49,7402fdc4,5a215e10,dcb01c35,4959b10c,fe31c7e9,d87ff33d,80fc06bd,8cc5b010,98088a19,50eed0db,1aa1329,67ab4722,35f56424,83b25eaf) @@ -209,7 +60,7 @@ static const secp256k1_ge_storage secp256k1_pre_g[ECMULT_TABLE_SIZE(WINDOW_G)] = ,S(754e3239,f325570c,dbbf4a87,deee8a66,b7f2b334,79d468fb,c1a50743,bf56cc18,673fb86,e5bda30f,b3cd0ed3,4ea49a0,23ee33d0,197a695d,c5d9809,3c536683) ,S(e3e6bd10,71a1e96a,ff57859c,82d570f0,33080066,1d1c952f,9fe26946,91d9b9e8,59c9e0bb,a394e76f,40c0aa58,379a3cb6,a5a22839,93e90c41,67002af4,920e37f5) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 32 +#if WINDOW_G > 7 ,S(186b483d,56a0338,26ae73d8,8f732985,c4ccb1f3,2ba35f4b,4cc47fdc,f04aa6eb,3b952d32,c67cf77e,2e17446e,204180ab,21fb8090,895138b4,a4a797f8,6e80888b) ,S(df9d70a6,b9876ce5,44c98561,f4be4f72,5442e6d2,b737d9c9,1a832172,4ce0963f,55eb2daf,d84d6ccd,5f862b78,5dc39d4a,b1572227,20ef9da2,17b8c45c,f2ba2417) ,S(5edd5cc2,3c51e87a,497ca815,d5dce0f8,ab52554f,849ed899,5de64c5f,34ce7143,efae9c8d,bc141306,61e8cec0,30c89ad0,c13c66c0,d17a2905,cdc706ab,7399a868) @@ -243,7 +94,7 @@ static const secp256k1_ge_storage secp256k1_pre_g[ECMULT_TABLE_SIZE(WINDOW_G)] = ,S(c4191636,5abb2b5d,9192f5f,2dbeafec,208f020f,12570a18,4dbadc3e,58595997,4f14351,d0087efa,49d245b3,28984989,d5caf945,f34bfc0,ed16e96b,58fa9913) ,S(841d6063,a586fa47,5a724604,da03bc5b,92a2e0d2,e0a36acf,e4c73a55,14742881,73867f5,9c0659e8,1904f9a1,c7543698,e62562d6,744c169c,e7a36de0,1a8d6154) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 64 +#if WINDOW_G > 8 ,S(5e95bb39,9a6971d3,76026947,f89bde2f,282b3381,928be4d,ed112ac4,d70e20d5,39f23f36,6809085b,eebfc711,81313775,a99c9aed,7d8ba38b,161384c7,46012865) ,S(36e4641a,53948fd4,76c39f8a,99fd974e,5ec07564,b5315d8b,f99471bc,a0ef2f66,d2424b1b,1abe4eb8,164227b0,85c9aa94,56ea1349,3fd563e0,6fd51cf5,694c78fc) ,S(336581e,a7bfbbb2,90c191a2,f507a41c,f5643842,170e914f,aeab27c2,c579f726,ead12168,595fe1be,99252129,b6e56b33,91f7ab14,10cd1e0e,f3dcdcab,d2fda224) @@ -309,7 +160,7 @@ static const secp256k1_ge_storage secp256k1_pre_g[ECMULT_TABLE_SIZE(WINDOW_G)] = ,S(809a20c6,7d64900f,fb698c4c,825f6d5f,2310fb04,51c86934,5b7319f6,45605721,9e994980,d9917e22,b76b0619,27fa0414,3d096ccc,54963e6a,5ebfa5f3,f8e286c1) ,S(1b38903a,43f7f114,ed4500b4,eac7083f,defece1c,f29c6352,8d563446,f972c180,4036edc9,31a60ae8,89353f77,fd53de4a,2708b26b,6f5da72a,d3394119,daf408f9) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 128 +#if WINDOW_G > 9 ,S(90a80db6,eb294b9e,ab0b4e8d,dfa3efe7,263458ce,2d07566d,f4e6c588,68feef23,753c8b9f,9754f18d,87f21145,d9e2936b,5ee050b2,7bbd9681,442c76e9,2fcf91e6) ,S(c2c80f84,4b705998,12d62546,f60340e,3e6f3605,4a14546e,6dc25d47,376bea9b,86ca160d,68f4d4e7,18b495b8,91d3b1b5,73b871a7,2b4cf61,23abd448,3aa79c64) ,S(9cf60674,4cf4b5f3,fdf989d3,f19fb265,2d00cfe1,d5fcd692,a323ce11,a28e7553,8147cbf7,b973fcc1,5b57b6a3,cfad6863,edd0f30e,3c45b85d,c300c513,c247759d) @@ -439,7 +290,7 @@ static const secp256k1_ge_storage secp256k1_pre_g[ECMULT_TABLE_SIZE(WINDOW_G)] = ,S(367807c9,a3606b4e,1b8c2616,ad528030,1dfcf686,40eddf02,fc59317c,230e9a86,1f023f2f,a2bbece7,3dba14c,124095cb,fdc4f92f,281a14,8304a412,c16ecae6) ,S(8ec4fdc3,9891f6af,1374e06f,c44b815,1b82541,75fc4909,acba5941,201af62b,2dc6cae5,cac2d887,83dca0e5,3c798f8,fe067bcf,5fc29751,13756cf7,ef4e5f1b) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 256 +#if WINDOW_G > 10 ,S(5cc24a6d,4c5b24f9,14542f91,e5fa937f,ffa08551,51b8b842,8729b06a,9178a263,b1da8635,81531a1f,bfb38a4e,419fa1fe,ca8d55a8,3ddbcb98,da19d5cf,fb7da472) ,S(83905926,c03905c3,a9644a6c,da810dd2,92602a50,50c52a21,9134fc4d,e3599e9f,4293260f,e8af6792,a20b115e,aa837638,9094298b,21d9de16,cf20e0c5,7a46089a) ,S(944b097e,4721e9dd,f8204ac3,d3878fa,e8fa6c14,34ae4822,481b2985,6589b6c7,5fc47565,30e9b095,f8b79643,1e745b99,1525bd4c,4764e8e,e8af4b96,9bd6ddf6) @@ -697,7 +548,7 @@ static const secp256k1_ge_storage secp256k1_pre_g[ECMULT_TABLE_SIZE(WINDOW_G)] = ,S(d9a0c689,95283291,972d6a72,897181b2,6b4ae317,4e98a676,f75bbfbf,81be876e,d36ae886,a0acbc9a,d1564d97,4d3f0309,e9039b93,bb350d92,2b7f8c7a,c6bfa042) ,S(c7a36324,6aeb7c8c,991b2aa7,10abdf5c,fff29912,30b3a69f,be2dd481,7c7c3e0a,1298fdd7,e448d2d,79986302,6b4b2d49,2b32148,d6d1e5f8,15f5b0c0,5c9e21f) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 512 +#if WINDOW_G > 11 ,S(635cd7a0,5064d3bc,66535a0,4dbf563a,640d2464,c7fe0ac4,8304214f,e4985a86,e40265,913e77f6,46735cfc,af9e2f30,e8d5f047,f3281a4c,e0453e27,e9e3ae1f) ,S(5da624f,38edea25,37f5b632,695b481a,eca54bb7,2169cadc,c5b91e10,989ee5f5,d3ea6dba,a1080300,fffaeeb2,43a5256e,48c28822,37b16505,a220d771,ca721779) ,S(aa4f64a2,b19775ec,92c1f687,1fe4b6d5,ce05278c,2976c80e,fe19284f,e87d4d5e,f39f117e,95fce0fb,6976e949,9583a66c,224ea028,8396518e,293793dc,3ed4f240) @@ -1211,7 +1062,7 @@ static const secp256k1_ge_storage secp256k1_pre_g[ECMULT_TABLE_SIZE(WINDOW_G)] = ,S(dd879eba,f870ee3d,f6e3f03,60833e12,fbdf844c,d6a5b76c,19802485,6649bbc1,e7bb04c3,a99a5c11,dd7a324c,1d5696c8,b041a12a,c6a538f7,3094716c,9943c55a) ,S(7aa4afbe,29b06a1e,da9319d5,72572b13,1df68f74,1b5f8d8e,d00ccc04,80f8016e,15f79a9f,b77b8587,7f02e3c1,3202b972,423fd00d,937c0d2d,c9d94b17,53fe7130) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 1024 +#if WINDOW_G > 12 ,S(9145f3f5,876a3265,5d7fee64,f2d6e660,c34354e9,1571d68b,a19e4cc5,8c39f890,45e0a95d,cf369fd0,5bc9ba7f,da108d6f,37650c1d,5766b6c9,98ecda28,b285d8ff) ,S(5450b752,36fb010d,1ef37afa,2077f3dc,a5a7f6c8,91a21317,13df740f,4511ac9e,a7c7ed57,62bef86d,4de1084d,3ded2d7b,13ff563a,67109ef0,8b6f0180,fe6ba175) ,S(ba6c0d72,5e48de2c,cb8256d6,7417d075,bbf2766f,d13501b8,4c32cf88,c0666a0f,cd2a9132,7137299f,50eda669,3a599032,bdecd64b,91bcc640,5ed186f3,e525e442) @@ -2237,7 +2088,7 @@ static const secp256k1_ge_storage secp256k1_pre_g[ECMULT_TABLE_SIZE(WINDOW_G)] = ,S(6d109d8c,2505808f,7b4052d6,b170ec80,c68373bb,e02f2b62,1cd29773,a96acb3e,c8dea0de,511f6e40,5141088e,cb023bff,200c870b,9cb952ad,c5038b28,eadc9a57) ,S(9c219898,6d202467,c055c734,73557505,6105b3e,2874dbf2,8f7f0c39,66e1af88,8637496c,8eddff12,f9dd4146,36565e0f,efddf5cd,52d48455,ec9fb2d,8dd0914a) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 2048 +#if WINDOW_G > 13 ,S(22ca5039,e660f60,1036dd75,2bb973b,dcd104b5,dcb8f2e7,4de8edc6,c292b03a,86fd4471,1ef6b81e,4416449a,708fa0c9,cb6731ba,c403e58e,da5e915f,646b11e8) ,S(cfc18f02,cc004640,f2116fdd,1f6ca202,2e39be25,df75e27c,80bde842,e6b6f938,d62b58df,4f79d0e5,97ba2923,f708cbe5,c09236a4,d9d01398,6451d684,6290df7d) ,S(3388bcc2,3425458e,991f46ca,6235c50a,57490556,becbaf25,9d3c14f9,9a80a087,7d4aa2f8,6f65783,1c22316,6958fbf3,6c3dca5,d4ac413b,3102e13e,a645c016) @@ -4287,7 +4138,7 @@ static const secp256k1_ge_storage secp256k1_pre_g[ECMULT_TABLE_SIZE(WINDOW_G)] = ,S(d5acff71,e82a455c,3a1ce292,689e8686,f441ce1b,e644e79a,bd6d0efe,29270865,aac6d48f,1b46e970,a044971a,ad13f033,ca8cde96,958d870e,dc7d80,1d26d5e8) ,S(5cf51c1b,2210d85,9d765e17,32109514,8f03fc57,51004b6a,91f098e2,e2711596,1eeb19e0,610df459,2c31e58e,aa2a2148,17fe9ee,a3995838,f395bdcf,26d5c3b3) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 4096 +#if WINDOW_G > 14 ,S(21456873,b58e3687,52c75800,8d3bcae,9efaf1f5,c5727842,25e3d854,8fd421cb,a2f2d10,c85e8a0f,4a136ad0,5df991ce,7d3c5585,a263d5cf,da50a4f4,3db76cb0) ,S(c10de5c0,51ae2d73,28ac06ca,cca840b4,ed7ab204,21c6122f,1d68fe7f,7893d38d,ee30e086,a891484,2ad4041,f9ab9c57,cff1a315,f0642d31,b31c2914,faac99e0) ,S(be778032,f12c1b77,9bba3d9e,d290ca90,30ac7050,bdd77a2,7eac09be,eea65c0b,8657348b,a1e27a63,1dd2a54b,e2d5270f,4cca817a,219c5378,4d4f73ba,2c932e63) @@ -8386,22 +8237,22 @@ static const secp256k1_ge_storage secp256k1_pre_g[ECMULT_TABLE_SIZE(WINDOW_G)] = ,S(1e70619c,381a6adc,e5d925e0,c9c74f97,3c02ff64,ff2662d7,34efc485,d2bce895,c923f771,f543ffed,42935c28,8474aaaf,80a46ad4,3c579ce0,bb5e663d,668b24b3) #endif }; -static const secp256k1_ge_storage secp256k1_pre_g_128[ECMULT_TABLE_SIZE(WINDOW_G)] = { +const secp256k1_ge_storage secp256k1_pre_g_128[ECMULT_TABLE_SIZE(WINDOW_G)] = { S(8f68b9d2,f63b5f33,9239c1ad,981f162e,e88c5678,723ea335,1b7b444c,9ec4c0da,662a9f2d,ba063986,de1d90c2,b6be215d,bbea2cfe,95510bfd,f23cbf79,501fff82) -#if ECMULT_TABLE_SIZE(WINDOW_G) > 1 +#if WINDOW_G > 2 ,S(38381dbe,2e509f22,8ba93363,f2451f08,fd845cb3,51d954be,18e2b8ed,d23809fa,e4a32d0a,fb917dc,b09405a5,520eb1cc,3681fccb,32d8f24d,bd707518,331fed52) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 2 +#if WINDOW_G > 3 ,S(49262724,e4372ae6,f6921b82,aa4699a1,f186aea5,40122630,3ea42648,97c2a310,1337e773,bca7abf9,5a2cfa56,9714303b,6d163612,a75ff8ce,c41b681,5e27ded0) ,S(e306568c,1a240c90,d5e253b3,e477e2f8,4dcc1a56,ff06db8d,1384b079,cebd2d31,eac6fe3,78934260,888f2b10,7f7d0db6,ffbc8042,be373826,692b4083,92546e44) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 4 +#if WINDOW_G > 4 ,S(3b9e100e,2428cefc,271b0e76,23fbd633,74ebf8d9,aab41dd9,c530c39e,363136b0,fafb9815,2d16bb71,df1533eb,8f475b26,a2ae28a3,3ad31f81,953ec16f,6cdbbc8a) ,S(bb0aad49,712ac9a9,2b76ca80,f5dedef7,17ca0768,8107beee,9608f047,2f485d3f,ea699c53,c5835479,8ecd201f,7297da34,895a5afa,31670bff,e7939250,3ca2f975) ,S(79090ac8,e4eefcc0,d4e8eb19,7afe0113,e1e58b4d,b01123de,4aeed33a,36718dc9,eaab722b,91905b8f,13d816cb,cd9aaa56,dd36afb7,ba9008b,963322b1,1cfae7c5) ,S(e77c81ad,e9f97b55,1c03dbbc,e549ba66,8dd71de7,cd775ad2,a269694c,7f60c7d1,3acf1478,eef81321,c5fc3b32,3ea81543,631470f7,1c2986d3,4ec581f2,82d72449) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 8 +#if WINDOW_G > 5 ,S(de2b5ce9,dbce511c,f2d8878e,3ded87cc,3d633dae,a2d45341,501fb3a4,55ccf6b0,f10576f3,d3c3e0e1,bbf717e9,8b1a3744,65b8c45a,c66318bb,34829eb7,11100666) ,S(d07bddff,d491a2fe,1ea59fbd,7c121217,29659ca5,de46658b,26b1460b,13c03c56,b2ad4708,cd3c97dd,f9c40e2,a1de04d5,61d963ff,8cc2eea7,6be3f60c,2b405ce7) ,S(82403e7c,5d3016af,3765ec4c,396ce8e1,f8da45c,434b8257,10edab41,bb6a4d51,d09661b,e27cb767,4456badd,b3e84051,99ab6ccc,4ec67c1b,11e92ead,7b463b19) @@ -8411,7 +8262,7 @@ static const secp256k1_ge_storage secp256k1_pre_g_128[ECMULT_TABLE_SIZE(WINDOW_G ,S(1f56f096,b18a7499,a153a5ae,acf8be05,8496dd23,da8e6c19,215628fb,c0567ed0,fef22b8a,3b52f490,83004436,b65cd69,c94189f4,1a93c0d5,1fc13cb4,379dff58) ,S(1d9f69b1,a4a47432,e386f525,234aa30,79e947cf,cf203297,4e0fc05b,638e213b,d898ec17,949c0761,b38500c3,a2b1da24,5438d5b3,d3f6f720,41f15d6e,e4d4ccbb) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 16 +#if WINDOW_G > 6 ,S(128c913,4d9dcb78,12fc4361,5c67ad0,55213354,dc8008b1,aeb5a9dc,fb629efd,fee3e54a,dd152610,d9725936,99d662,c160c8e4,ec6f76e4,5ff41818,be67c96) ,S(21ec012f,5a95b94d,244b8d51,9756075c,301f2854,8e2c51fc,49c0e3d9,d1a9685,2def2105,77af497f,4c7fef71,6949f28e,7418eda6,fd5fc162,d128de19,3cde08ae) ,S(688f5202,fb9d8bc0,9e480e89,4c7cfc74,761c3be7,7dafb11c,58422836,3e331cc5,96ba7d59,63b541a7,2ec7cabd,92403434,1a393eaf,89eebd94,62d9c218,c7302cd3) @@ -8429,7 +8280,7 @@ static const secp256k1_ge_storage secp256k1_pre_g_128[ECMULT_TABLE_SIZE(WINDOW_G ,S(144e88f6,3e73abff,72cac11e,7ddccf79,19e744e6,278941ae,18d1b797,e098e4e5,63cdbf3,5df3c655,c58197f,ea54633d,158705cf,7dc2eb3b,4e09f83c,3021837c) ,S(9436e3dc,489ecd8e,2d16a739,c9c73e3d,60e5bc93,68157039,75b8efbd,5c3a9081,1460531f,50cb6ebf,d1aa7806,ea84e7f7,8e8d76b2,b3a66d5e,3a0bf60,39a7e59c) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 32 +#if WINDOW_G > 7 ,S(9d3c2561,7a56d10b,46d9b01a,1710d193,e840e005,df669e76,1936c275,20890db9,6bbdc0bc,4c4ae9bc,c2dfee9b,82da9b94,1f89ffcd,e8af2aca,4467ce3,78521ea3) ,S(29f98e50,f51b7f8b,e18c6ae0,b453c4f2,d0aca5a8,b0e61d2d,dda8506,3fdb76c8,daf3bcdc,ae8e031c,73eb8b21,14058063,58a6ec30,ad379186,df80e3c7,f0e5d28f) ,S(d67d30c2,c71daa36,1805e31,1dd6046e,17a89752,94d76e1a,538af074,4dc22c94,48b9b0e7,12c807b0,b92e690a,a2e068cc,e87ebbbe,aaf4bd96,9c1114bb,a54f670c) @@ -8463,7 +8314,7 @@ static const secp256k1_ge_storage secp256k1_pre_g_128[ECMULT_TABLE_SIZE(WINDOW_G ,S(6f97ac0f,27b87905,6b442d13,e566978e,91f0cc1d,d6ac1e64,e9764a35,325dd1b7,83c6e70c,fac6c707,226ce1cc,691b38a0,7e937f5a,5f2d9c81,4dd0d3ff,9f433d32) ,S(72c5d60f,eb014e2d,ba8265ca,d454f261,2d6abcd4,b2236bad,c94c4801,561dce1f,e3119a19,7ef91963,b3b28216,3c5d3acb,97b281b6,d246cbf0,690b40da,63978fe2) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 64 +#if WINDOW_G > 8 ,S(a96d2da0,1b10186,6998659d,f441a1b0,2af32b94,aae8c6ea,707d9ed0,d5f33825,660d7d83,5da5235,9f7cfd41,28c370aa,5659ea71,16a91690,6c0e8108,a513f9f) ,S(2cce6f63,4d815ecc,1981d200,87616677,d906aa27,990c4875,17314dc5,5be3c4fa,615210dd,bd599e91,1b6f997a,fd05475,b33cb274,c9ecb6e5,d3c23323,beba4b50) ,S(992b0084,525dd399,d98602d9,8b8d53b2,4558fafc,758a2f46,60e89bd6,a645f0c4,83ca98d2,26545a29,8c45f40b,11420602,f5a5c70e,595eea57,2da64d61,a4e2f98d) @@ -8529,7 +8380,7 @@ static const secp256k1_ge_storage secp256k1_pre_g_128[ECMULT_TABLE_SIZE(WINDOW_G ,S(7590a4a,627d5e90,e42a4b0f,6be6497a,f906182d,d2dddc48,a8b6bbfd,f56c0504,d611e21a,b5498760,5c97e878,baa464bc,cc5bd875,353b69f8,1f93ce2a,a6c38587) ,S(94b1da0f,b310e056,db0dff72,81db3362,1fcc555d,bf3c973b,76097908,7fe19d6a,8318893d,d5b41a56,33a2ab4,ae4b953c,45c42e9c,8f2fd159,85286de3,fd4fc217) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 128 +#if WINDOW_G > 9 ,S(b5af4299,bcdacef1,e07d081,daec2cfa,d6f8f821,38e151a5,f20e6d52,84c9a6cb,c0984407,8a7db82d,f572987e,b137dc09,c8cf65fd,aedcb20,43b2479b,ab95448d) ,S(5f49ad43,70744587,3980a153,c851829b,f8ef6141,9889bb0c,3c476847,6939c3e3,5c40d385,20f56c3d,ba08ae1,b40fc24a,2ae25c94,45cae0f7,d01d1800,747e04eb) ,S(f6bb067f,88ccc11c,64e30d1a,e6893942,16bea3bf,26ec9c64,5cde1b9e,487da385,315a476d,7268978c,d89d4ec8,adde4a83,28dbfdd9,f2bf44fe,ad4ed721,78288f55) @@ -8659,7 +8510,7 @@ static const secp256k1_ge_storage secp256k1_pre_g_128[ECMULT_TABLE_SIZE(WINDOW_G ,S(8ac79852,75673818,69fe93c9,6141493e,72ca344,790487b,d5425ed6,9f5c5c18,bb314248,fcbfc867,d1972e04,b9ef1f90,775375f9,9d25bcec,684c72c9,bdd1c08e) ,S(560e9711,88cbc7ce,c61f3bf4,45f0ade3,6f3f174d,219a160,5f9c8692,3f848b9d,9e92dace,6775cc67,bfbbf21f,c64e6e9d,4d133f8a,c18ee277,bc80ef6d,d1b9cd2f) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 256 +#if WINDOW_G > 10 ,S(8c464204,4b95eb66,cc95ca08,8469a1c0,28eea52f,7f709fe,e6d90700,f5fb943a,bf10fc5d,7bc25181,301b3a61,5ebea597,a3492025,953c3aa5,1a11271e,689d0223) ,S(d293f74c,f3578b71,35377247,cd8b0367,7f2245da,f87453cb,4d8d1cc2,871eb5aa,86c2013a,61dfea14,63e45931,eb09050f,f080e3d4,ae357423,ba7afbed,a64a6f26) ,S(35997241,ae757b1f,c767611e,c76eb935,fefbf7a7,33666aff,4c6bd744,d7687d35,bcbea61f,246bcd4c,c3c7fd35,7bd393da,2f36e0ef,cca0df9c,5994f96f,c44b2aa0) @@ -8917,7 +8768,7 @@ static const secp256k1_ge_storage secp256k1_pre_g_128[ECMULT_TABLE_SIZE(WINDOW_G ,S(6ca7e032,8bf5d67a,b804d431,e5f709e,bd3156e0,1d4511da,1dc67394,24d2e659,c428b133,f3683909,6551c2ff,2f870d80,d80aaaf4,6d2b1a69,7722057,5eca2647) ,S(60df19a9,83e9086c,4cbd20ba,fbafa8b2,346ae4ee,9c1ad4dd,99959e91,2df530d3,dae7b854,29f817a0,421c2f1e,e4e8e4d,a09d60f8,84701e31,d7a8b7b0,3b79cc48) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 512 +#if WINDOW_G > 11 ,S(6f18d50e,5ef5f2ad,bad80ea1,c59a3847,5c22ff05,6ba52c7,e1d26d6d,d3686bce,d1d7ea0f,a166efd0,facf5c83,bc3786e0,d3f3405d,5b4578f5,13a336ad,f0d7d7a9) ,S(b634dc9d,5ed51336,4ac9569a,8ddee9d9,fcdfe00e,65e59233,b3be8a07,2fd949e0,d72e46e2,c61b2575,f25a505c,bdc3456e,fdf18976,6562a1a7,e86d036,eb31db69) ,S(5a37fcf0,2dba24f3,e6646171,43dbc5ab,40d91983,69f5cf0e,4fc566fd,97445910,a960d1d8,13f8746c,662a9582,614c7847,33e6153d,2975cb59,c3342463,75c69d1e) @@ -9431,7 +9282,7 @@ static const secp256k1_ge_storage secp256k1_pre_g_128[ECMULT_TABLE_SIZE(WINDOW_G ,S(edc9e8b1,3c61a1e0,8f93be12,495084ed,b5a90eca,4f11dc03,cd217b1a,56285a2c,66c2fd21,7e8fac7,4fd43b1b,58c80661,b0e8df85,a2fddfda,5e9f99fe,621e1f3d) ,S(dd94295f,8388a498,2aaa4164,927b81ba,cae9d002,b3ddb6b6,97082c70,f1e2c66e,838c060c,d409c1ea,9bcd9e8c,b1fac83f,3b08c2b2,482d8f4b,fd3ebc18,8d6706b) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 1024 +#if WINDOW_G > 12 ,S(7e068e04,dfc0be48,e5c0d3b3,5bfc734e,96e96ddd,d0ac4876,92f74535,685ab7e,df2cd146,90d225c8,d04052e6,93f14bf,69351e08,79883646,2c88401e,4ec70d0a) ,S(73a9bfcf,d10aac9b,46e659f,76c439a0,b7c2a073,7dec217a,21f43f39,949a5052,73b91529,3ea7b052,682062c,8f86cbf,bf379df6,91a9cec2,4a1424a9,b3be10dd) ,S(efddff84,c8cc754,e6e34678,d85809bc,55cc224b,f69be05a,daa847ec,d408c55b,65dd8f41,8264aab2,efe6ed7e,c45b4ac,8aac218a,3b6713fc,1736dd6d,c2f188d5) @@ -10457,7 +10308,7 @@ static const secp256k1_ge_storage secp256k1_pre_g_128[ECMULT_TABLE_SIZE(WINDOW_G ,S(8a663e7,40cad237,12dad6f9,1bb5d266,30542933,853e02eb,665e198d,a5c6e53b,e206b399,51175d15,daccf711,63b0918c,786a8fe,2077812,c0c1eee6,bb5c1e99) ,S(2b0956cb,e0ca91ee,f44fac0a,76cc0b00,5f4a5ae8,27e63696,fe1aa8b4,a92941a1,a31e6fa8,dd454eaf,90ee1e11,62627e3,3f84b8fe,da29f423,ddf2a962,386cffd2) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 2048 +#if WINDOW_G > 13 ,S(e0144151,7a2ac970,eb6381df,7e11fb4c,6b009980,6a0d330,d5429126,e662c3bd,8238ce9b,a691c80c,21563934,18d18dbf,aeb3fd34,48863a1e,7f4ba360,408dfcee) ,S(be3213e,8b062f33,caf16c53,5a0b666,f344e0d7,1e28aeba,8b215a3,7ec86c37,552523ba,eaca38a4,cd795683,852a2643,550fb83d,d1db0adc,3f1b29c9,7d51cd1f) ,S(623393b6,bfbfbd64,fc7aa1db,9e58e274,1c18eb6d,b5eb30ab,c4fe167e,a9e8ff2b,7c0e4174,f0fd5bf2,a025e316,fa3b7b97,1339a197,b52b0b50,bad4dfe,34e42cd3) @@ -12507,7 +12358,7 @@ static const secp256k1_ge_storage secp256k1_pre_g_128[ECMULT_TABLE_SIZE(WINDOW_G ,S(4beaa213,3c6161d0,c7da6d7c,8eb09cba,2e8ca505,a790a7d3,9d0c16b,8053bb52,da4c083c,9cbdda2e,c6626d8f,e4b2ee13,703496c,c298bffa,db1acdca,99d4f6d5) ,S(a8d31ec6,53ce1834,a78de363,c5f9abd1,8594b34,4d34f5fc,8a10e81f,5804831,b4d6a9cd,7cbbe370,9edb5601,cb7c8ba4,8c462c18,594a4bce,64f4c286,dac5cff9) #endif -#if ECMULT_TABLE_SIZE(WINDOW_G) > 4096 +#if WINDOW_G > 14 ,S(a8f11586,f3df4945,a753c485,ee0fd4d,e410771d,bddc1b26,c9ff10e0,77b915b7,a4ad6f16,dbd741bc,71b2dfd2,dd2d340,3816bf73,4e73cc10,abcfa6bb,d0f161a4) ,S(13e697c3,812d4772,254082da,372892d4,e1a66e1a,eb16bbd4,f7a0d531,c979cb2,87fa7baa,f6def12d,31e42c14,f672c0d6,a9d0e2e1,ceee2546,d65bd01,c18df57f) ,S(3cae4590,821a9697,a5963269,b44f0222,98f60021,b6048b3f,49c6fd4d,8650c7a,e03f8745,60418449,ad97f28,41664745,349329d,268c1c43,86e25147,6e44b234) @@ -16606,6 +16457,4 @@ static const secp256k1_ge_storage secp256k1_pre_g_128[ECMULT_TABLE_SIZE(WINDOW_G ,S(1b9a142f,fc4d03ea,4b079f2d,b05fad98,8ddb2d32,b359967f,c173801f,63320825,59bda7ed,5b691c20,4fc8f8ac,f53be298,ae628954,a8134d0f,dd097e67,be9ff9b6) #endif }; -#endif #undef S -#endif diff --git a/src/secp256k1/src/precomputed_ecmult.h b/src/secp256k1/src/precomputed_ecmult.h new file mode 100644 index 0000000000..949b62c874 --- /dev/null +++ b/src/secp256k1/src/precomputed_ecmult.h @@ -0,0 +1,35 @@ +/***************************************************************************************************** + * Copyright (c) 2013, 2014, 2017, 2021 Pieter Wuille, Andrew Poelstra, Jonas Nick, Russell O'Connor * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or https://www.opensource.org/licenses/mit-license.php. * + *****************************************************************************************************/ + +#ifndef SECP256K1_PRECOMPUTED_ECMULT_H +#define SECP256K1_PRECOMPUTED_ECMULT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "group.h" +#if defined(EXHAUSTIVE_TEST_ORDER) +#if EXHAUSTIVE_TEST_ORDER == 13 +# define WINDOW_G 4 +# elif EXHAUSTIVE_TEST_ORDER == 199 +# define WINDOW_G 8 +# else +# error No known generator for the specified exhaustive test group order. +# endif +static secp256k1_ge_storage secp256k1_pre_g[ECMULT_TABLE_SIZE(WINDOW_G)]; +static secp256k1_ge_storage secp256k1_pre_g_128[ECMULT_TABLE_SIZE(WINDOW_G)]; +#else /* !defined(EXHAUSTIVE_TEST_ORDER) */ +# define WINDOW_G ECMULT_WINDOW_SIZE +extern const secp256k1_ge_storage secp256k1_pre_g[ECMULT_TABLE_SIZE(WINDOW_G)]; +extern const secp256k1_ge_storage secp256k1_pre_g_128[ECMULT_TABLE_SIZE(WINDOW_G)]; +#endif /* defined(EXHAUSTIVE_TEST_ORDER) */ + +#ifdef __cplusplus +} +#endif + +#endif /* SECP256K1_PRECOMPUTED_ECMULT_H */ diff --git a/src/secp256k1/src/ecmult_gen_static_prec_table.h b/src/secp256k1/src/precomputed_ecmult_gen.c index bf4e5ea248..d67291fcf5 100644 --- a/src/secp256k1/src/ecmult_gen_static_prec_table.h +++ b/src/secp256k1/src/precomputed_ecmult_gen.c @@ -1,13 +1,17 @@ -/* This file was automatically generated by gen_ecmult_gen_static_prec_table. */ +/* This file was automatically generated by precompute_ecmult_gen. */ /* See ecmult_gen_impl.h for details about the contents of this file. */ -#ifndef SECP256K1_ECMULT_GEN_STATIC_PREC_TABLE_H -#define SECP256K1_ECMULT_GEN_STATIC_PREC_TABLE_H +#if defined HAVE_CONFIG_H +# include "libsecp256k1-config.h" +#endif +#include "../include/secp256k1.h" #include "group.h" -#define S(a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p) SECP256K1_GE_STORAGE_CONST(0x##a##u,0x##b##u,0x##c##u,0x##d##u,0x##e##u,0x##f##u,0x##g##u,0x##h##u,0x##i##u,0x##j##u,0x##k##u,0x##l##u,0x##m##u,0x##n##u,0x##o##u,0x##p##u) +#include "ecmult_gen.h" +#include "precomputed_ecmult_gen.h" #ifdef EXHAUSTIVE_TEST_ORDER -static secp256k1_ge_storage secp256k1_ecmult_gen_prec_table[ECMULT_GEN_PREC_N(ECMULT_GEN_PREC_BITS)][ECMULT_GEN_PREC_G(ECMULT_GEN_PREC_BITS)]; -#else -static const secp256k1_ge_storage secp256k1_ecmult_gen_prec_table[ECMULT_GEN_PREC_N(ECMULT_GEN_PREC_BITS)][ECMULT_GEN_PREC_G(ECMULT_GEN_PREC_BITS)] = { +# error Cannot compile precomputed_ecmult_gen.c in exhaustive test mode +#endif /* EXHAUSTIVE_TEST_ORDER */ +#define S(a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p) SECP256K1_GE_STORAGE_CONST(0x##a##u,0x##b##u,0x##c##u,0x##d##u,0x##e##u,0x##f##u,0x##g##u,0x##h##u,0x##i##u,0x##j##u,0x##k##u,0x##l##u,0x##m##u,0x##n##u,0x##o##u,0x##p##u) +const secp256k1_ge_storage secp256k1_ecmult_gen_prec_table[ECMULT_GEN_PREC_N(ECMULT_GEN_PREC_BITS)][ECMULT_GEN_PREC_G(ECMULT_GEN_PREC_BITS)] = { #if ECMULT_GEN_PREC_BITS == 2 {S(3a9ed373,6eed3eec,9aeb5ac0,21b54652,56817b1f,8de6cd0,fbcee548,ba044bb5,7bcc5928,bdc9c023,dfc663b8,9e4f6969,ab751798,8e600ec1,d242010c,45c7974a), S(e44d7675,c3cb2857,4e133c01,a74f4afc,5ce684f8,4a789711,603f7c4f,50abef58,25bcb62f,fe2e2ce2,196ad86c,a006e20,8c64d21b,b25320a3,b5574b9c,1e1bfb4b), @@ -9743,6 +9747,4 @@ S(244b87a4,fcecef37,76c16c5c,24c7785,be3b3c13,46595363,b8c066ec,45bfe561,9642f5f S(9de52b81,157165cc,aef44485,4c2b3535,a599a79,80d024de,5334b385,ecbb2e91,74fca165,26fe2f87,a41ce510,4dd5634,5cf98c11,803c0392,3eb4b8b7,60240c02)} #endif }; -#endif /* EXHAUSTIVE_TEST_ORDER */ -#undef SC -#endif /* SECP256K1_ECMULT_GEN_STATIC_PREC_TABLE_H */ +#undef S diff --git a/src/secp256k1/src/precomputed_ecmult_gen.h b/src/secp256k1/src/precomputed_ecmult_gen.h new file mode 100644 index 0000000000..7256ad2e30 --- /dev/null +++ b/src/secp256k1/src/precomputed_ecmult_gen.h @@ -0,0 +1,26 @@ +/********************************************************************************* + * Copyright (c) 2013, 2014, 2015, 2021 Thomas Daede, Cory Fields, Pieter Wuille * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or https://www.opensource.org/licenses/mit-license.php. * + *********************************************************************************/ + +#ifndef SECP256K1_PRECOMPUTED_ECMULT_GEN_H +#define SECP256K1_PRECOMPUTED_ECMULT_GEN_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "group.h" +#include "ecmult_gen.h" +#ifdef EXHAUSTIVE_TEST_ORDER +static secp256k1_ge_storage secp256k1_ecmult_gen_prec_table[ECMULT_GEN_PREC_N(ECMULT_GEN_PREC_BITS)][ECMULT_GEN_PREC_G(ECMULT_GEN_PREC_BITS)]; +#else +extern const secp256k1_ge_storage secp256k1_ecmult_gen_prec_table[ECMULT_GEN_PREC_N(ECMULT_GEN_PREC_BITS)][ECMULT_GEN_PREC_G(ECMULT_GEN_PREC_BITS)]; +#endif /* defined(EXHAUSTIVE_TEST_ORDER) */ + +#ifdef __cplusplus +} +#endif + +#endif /* SECP256K1_PRECOMPUTED_ECMULT_GEN_H */ diff --git a/src/secp256k1/src/secp256k1.c b/src/secp256k1/src/secp256k1.c index 36fde24c3d..8f34c35283 100644 --- a/src/secp256k1/src/secp256k1.c +++ b/src/secp256k1/src/secp256k1.c @@ -423,8 +423,12 @@ static int nonce_function_rfc6979(unsigned char *nonce32, const unsigned char *m unsigned int offset = 0; secp256k1_rfc6979_hmac_sha256 rng; unsigned int i; + secp256k1_scalar msg; + unsigned char msgmod32[32]; + secp256k1_scalar_set_b32(&msg, msg32, NULL); + secp256k1_scalar_get_b32(msgmod32, &msg); /* We feed a byte array to the PRNG as input, consisting of: - * - the private key (32 bytes) and message (32 bytes), see RFC 6979 3.2d. + * - the private key (32 bytes) and reduced message (32 bytes), see RFC 6979 3.2d. * - optionally 32 extra bytes of data, see RFC 6979 3.6 Additional Data. * - optionally 16 extra bytes with the algorithm name. * Because the arguments have distinct fixed lengths it is not possible for @@ -432,7 +436,7 @@ static int nonce_function_rfc6979(unsigned char *nonce32, const unsigned char *m * nonces. */ buffer_append(keydata, &offset, key32, 32); - buffer_append(keydata, &offset, msg32, 32); + buffer_append(keydata, &offset, msgmod32, 32); if (data != NULL) { buffer_append(keydata, &offset, data, 32); } diff --git a/src/secp256k1/src/testrand.h b/src/secp256k1/src/testrand.h index 667d1867bd..bd149bb1b4 100644 --- a/src/secp256k1/src/testrand.h +++ b/src/secp256k1/src/testrand.h @@ -17,11 +17,14 @@ SECP256K1_INLINE static void secp256k1_testrand_seed(const unsigned char *seed16); /** Generate a pseudorandom number in the range [0..2**32-1]. */ -static uint32_t secp256k1_testrand32(void); +SECP256K1_INLINE static uint32_t secp256k1_testrand32(void); + +/** Generate a pseudorandom number in the range [0..2**64-1]. */ +SECP256K1_INLINE static uint64_t secp256k1_testrand64(void); /** Generate a pseudorandom number in the range [0..2**bits-1]. Bits must be 1 or * more. */ -static uint32_t secp256k1_testrand_bits(int bits); +SECP256K1_INLINE static uint64_t secp256k1_testrand_bits(int bits); /** Generate a pseudorandom number in the range [0..range-1]. */ static uint32_t secp256k1_testrand_int(uint32_t range); diff --git a/src/secp256k1/src/testrand_impl.h b/src/secp256k1/src/testrand_impl.h index c8d30ef6a8..e9b9d7ded4 100644 --- a/src/secp256k1/src/testrand_impl.h +++ b/src/secp256k1/src/testrand_impl.h @@ -14,37 +14,64 @@ #include "testrand.h" #include "hash.h" -static secp256k1_rfc6979_hmac_sha256 secp256k1_test_rng; -static uint32_t secp256k1_test_rng_precomputed[8]; -static int secp256k1_test_rng_precomputed_used = 8; +static uint64_t secp256k1_test_state[4]; static uint64_t secp256k1_test_rng_integer; static int secp256k1_test_rng_integer_bits_left = 0; SECP256K1_INLINE static void secp256k1_testrand_seed(const unsigned char *seed16) { - secp256k1_rfc6979_hmac_sha256_initialize(&secp256k1_test_rng, seed16, 16); + static const unsigned char PREFIX[19] = "secp256k1 test init"; + unsigned char out32[32]; + secp256k1_sha256 hash; + int i; + + /* Use SHA256(PREFIX || seed16) as initial state. */ + secp256k1_sha256_initialize(&hash); + secp256k1_sha256_write(&hash, PREFIX, sizeof(PREFIX)); + secp256k1_sha256_write(&hash, seed16, 16); + secp256k1_sha256_finalize(&hash, out32); + for (i = 0; i < 4; ++i) { + uint64_t s = 0; + int j; + for (j = 0; j < 8; ++j) s = (s << 8) | out32[8*i + j]; + secp256k1_test_state[i] = s; + } + secp256k1_test_rng_integer_bits_left = 0; } -SECP256K1_INLINE static uint32_t secp256k1_testrand32(void) { - if (secp256k1_test_rng_precomputed_used == 8) { - secp256k1_rfc6979_hmac_sha256_generate(&secp256k1_test_rng, (unsigned char*)(&secp256k1_test_rng_precomputed[0]), sizeof(secp256k1_test_rng_precomputed)); - secp256k1_test_rng_precomputed_used = 0; - } - return secp256k1_test_rng_precomputed[secp256k1_test_rng_precomputed_used++]; +SECP256K1_INLINE static uint64_t rotl(const uint64_t x, int k) { + return (x << k) | (x >> (64 - k)); +} + +SECP256K1_INLINE static uint64_t secp256k1_testrand64(void) { + /* Test-only Xoshiro256++ RNG. See https://prng.di.unimi.it/ */ + const uint64_t result = rotl(secp256k1_test_state[0] + secp256k1_test_state[3], 23) + secp256k1_test_state[0]; + const uint64_t t = secp256k1_test_state[1] << 17; + secp256k1_test_state[2] ^= secp256k1_test_state[0]; + secp256k1_test_state[3] ^= secp256k1_test_state[1]; + secp256k1_test_state[1] ^= secp256k1_test_state[2]; + secp256k1_test_state[0] ^= secp256k1_test_state[3]; + secp256k1_test_state[2] ^= t; + secp256k1_test_state[3] = rotl(secp256k1_test_state[3], 45); + return result; } -static uint32_t secp256k1_testrand_bits(int bits) { - uint32_t ret; +SECP256K1_INLINE static uint64_t secp256k1_testrand_bits(int bits) { + uint64_t ret; if (secp256k1_test_rng_integer_bits_left < bits) { - secp256k1_test_rng_integer |= (((uint64_t)secp256k1_testrand32()) << secp256k1_test_rng_integer_bits_left); - secp256k1_test_rng_integer_bits_left += 32; + secp256k1_test_rng_integer = secp256k1_testrand64(); + secp256k1_test_rng_integer_bits_left = 64; } ret = secp256k1_test_rng_integer; secp256k1_test_rng_integer >>= bits; secp256k1_test_rng_integer_bits_left -= bits; - ret &= ((~((uint32_t)0)) >> (32 - bits)); + ret &= ((~((uint64_t)0)) >> (64 - bits)); return ret; } +SECP256K1_INLINE static uint32_t secp256k1_testrand32(void) { + return secp256k1_testrand_bits(32); +} + static uint32_t secp256k1_testrand_int(uint32_t range) { /* We want a uniform integer between 0 and range-1, inclusive. * B is the smallest number such that range <= 2**B. @@ -85,7 +112,19 @@ static uint32_t secp256k1_testrand_int(uint32_t range) { } static void secp256k1_testrand256(unsigned char *b32) { - secp256k1_rfc6979_hmac_sha256_generate(&secp256k1_test_rng, b32, 32); + int i; + for (i = 0; i < 4; ++i) { + uint64_t val = secp256k1_testrand64(); + b32[0] = val; + b32[1] = val >> 8; + b32[2] = val >> 16; + b32[3] = val >> 24; + b32[4] = val >> 32; + b32[5] = val >> 40; + b32[6] = val >> 48; + b32[7] = val >> 56; + b32 += 8; + } } static void secp256k1_testrand_bytes_test(unsigned char *bytes, size_t len) { @@ -109,7 +148,7 @@ static void secp256k1_testrand256_test(unsigned char *b32) { } static void secp256k1_testrand_flip(unsigned char *b, size_t len) { - b[secp256k1_testrand_int(len)] ^= (1 << secp256k1_testrand_int(8)); + b[secp256k1_testrand_int(len)] ^= (1 << secp256k1_testrand_bits(3)); } static void secp256k1_testrand_init(const char* hexseed) { diff --git a/src/secp256k1/src/tests.c b/src/secp256k1/src/tests.c index 712fc655fa..dd53173930 100644 --- a/src/secp256k1/src/tests.c +++ b/src/secp256k1/src/tests.c @@ -28,6 +28,8 @@ #include "modinv64_impl.h" #endif +#define CONDITIONAL_TEST(cnt, nam) if (count < (cnt)) { printf("Skipping %s (iteration count too low)\n", nam); } else + static int count = 64; static secp256k1_context *ctx = NULL; @@ -100,6 +102,12 @@ void random_group_element_jacobian_test(secp256k1_gej *gej, const secp256k1_ge * gej->infinity = ge->infinity; } +void random_gej_test(secp256k1_gej *gej) { + secp256k1_ge ge; + random_group_element_test(&ge); + random_group_element_jacobian_test(gej, &ge); +} + void random_scalar_order_test(secp256k1_scalar *num) { do { unsigned char b32[32]; @@ -443,14 +451,18 @@ void run_ctz_tests(void) { /***** HASH TESTS *****/ -void run_sha256_tests(void) { - static const char *inputs[8] = { +void run_sha256_known_output_tests(void) { + static const char *inputs[] = { "", "abc", "message digest", "secure hash algorithm", "SHA256 is considered to be safe", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", "For this sample, this 63-byte string will be used as input data", - "This is exactly 64 bytes long, not counting the terminating byte" + "This is exactly 64 bytes long, not counting the terminating byte", + "aaaaa", }; - static const unsigned char outputs[8][32] = { + static const unsigned int repeat[] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1000000/5 + }; + static const unsigned char outputs[][32] = { {0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}, {0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c, 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad}, {0xf7, 0x84, 0x6f, 0x55, 0xcf, 0x23, 0xe1, 0x4e, 0xeb, 0xea, 0xb5, 0xb4, 0xe1, 0x55, 0x0c, 0xad, 0x5b, 0x50, 0x9e, 0x33, 0x48, 0xfb, 0xc4, 0xef, 0xa3, 0xa1, 0x41, 0x3d, 0x39, 0x3c, 0xb6, 0x50}, @@ -458,27 +470,146 @@ void run_sha256_tests(void) { {0x68, 0x19, 0xd9, 0x15, 0xc7, 0x3f, 0x4d, 0x1e, 0x77, 0xe4, 0xe1, 0xb5, 0x2d, 0x1f, 0xa0, 0xf9, 0xcf, 0x9b, 0xea, 0xea, 0xd3, 0x93, 0x9f, 0x15, 0x87, 0x4b, 0xd9, 0x88, 0xe2, 0xa2, 0x36, 0x30}, {0x24, 0x8d, 0x6a, 0x61, 0xd2, 0x06, 0x38, 0xb8, 0xe5, 0xc0, 0x26, 0x93, 0x0c, 0x3e, 0x60, 0x39, 0xa3, 0x3c, 0xe4, 0x59, 0x64, 0xff, 0x21, 0x67, 0xf6, 0xec, 0xed, 0xd4, 0x19, 0xdb, 0x06, 0xc1}, {0xf0, 0x8a, 0x78, 0xcb, 0xba, 0xee, 0x08, 0x2b, 0x05, 0x2a, 0xe0, 0x70, 0x8f, 0x32, 0xfa, 0x1e, 0x50, 0xc5, 0xc4, 0x21, 0xaa, 0x77, 0x2b, 0xa5, 0xdb, 0xb4, 0x06, 0xa2, 0xea, 0x6b, 0xe3, 0x42}, - {0xab, 0x64, 0xef, 0xf7, 0xe8, 0x8e, 0x2e, 0x46, 0x16, 0x5e, 0x29, 0xf2, 0xbc, 0xe4, 0x18, 0x26, 0xbd, 0x4c, 0x7b, 0x35, 0x52, 0xf6, 0xb3, 0x82, 0xa9, 0xe7, 0xd3, 0xaf, 0x47, 0xc2, 0x45, 0xf8} + {0xab, 0x64, 0xef, 0xf7, 0xe8, 0x8e, 0x2e, 0x46, 0x16, 0x5e, 0x29, 0xf2, 0xbc, 0xe4, 0x18, 0x26, 0xbd, 0x4c, 0x7b, 0x35, 0x52, 0xf6, 0xb3, 0x82, 0xa9, 0xe7, 0xd3, 0xaf, 0x47, 0xc2, 0x45, 0xf8}, + {0xcd, 0xc7, 0x6e, 0x5c, 0x99, 0x14, 0xfb, 0x92, 0x81, 0xa1, 0xc7, 0xe2, 0x84, 0xd7, 0x3e, 0x67, 0xf1, 0x80, 0x9a, 0x48, 0xa4, 0x97, 0x20, 0x0e, 0x04, 0x6d, 0x39, 0xcc, 0xc7, 0x11, 0x2c, 0xd0}, }; - int i; - for (i = 0; i < 8; i++) { + unsigned int i, ninputs; + + /* Skip last input vector for low iteration counts */ + ninputs = sizeof(inputs)/sizeof(inputs[0]) - 1; + CONDITIONAL_TEST(16, "run_sha256_known_output_tests 1000000") ninputs++; + + for (i = 0; i < ninputs; i++) { unsigned char out[32]; secp256k1_sha256 hasher; + unsigned int j; + /* 1. Run: simply write the input bytestrings */ + j = repeat[i]; secp256k1_sha256_initialize(&hasher); - secp256k1_sha256_write(&hasher, (const unsigned char*)(inputs[i]), strlen(inputs[i])); + while (j > 0) { + secp256k1_sha256_write(&hasher, (const unsigned char*)(inputs[i]), strlen(inputs[i])); + j--; + } secp256k1_sha256_finalize(&hasher, out); CHECK(secp256k1_memcmp_var(out, outputs[i], 32) == 0); + /* 2. Run: split the input bytestrings randomly before writing */ if (strlen(inputs[i]) > 0) { int split = secp256k1_testrand_int(strlen(inputs[i])); secp256k1_sha256_initialize(&hasher); - secp256k1_sha256_write(&hasher, (const unsigned char*)(inputs[i]), split); - secp256k1_sha256_write(&hasher, (const unsigned char*)(inputs[i] + split), strlen(inputs[i]) - split); + j = repeat[i]; + while (j > 0) { + secp256k1_sha256_write(&hasher, (const unsigned char*)(inputs[i]), split); + secp256k1_sha256_write(&hasher, (const unsigned char*)(inputs[i] + split), strlen(inputs[i]) - split); + j--; + } secp256k1_sha256_finalize(&hasher, out); CHECK(secp256k1_memcmp_var(out, outputs[i], 32) == 0); } } } +/** SHA256 counter tests + +The tests verify that the SHA256 counter doesn't wrap around at message length +2^i bytes for i = 20, ..., 33. This wide range aims at being independent of the +implementation of the counter and it catches multiple natural 32-bit overflows +(e.g., counting bits, counting bytes, counting blocks, ...). + +The test vectors have been generated using following Python script which relies +on https://github.com/cloudtools/sha256/ (v0.3 on Python v3.10.2). + +``` +from sha256 import sha256 +from copy import copy + +def midstate_c_definition(hasher): + ret = ' {{0x' + hasher.state[0].hex('_', 4).replace('_', ', 0x') + '},\n' + ret += ' {0x00}, ' + str(hex(hasher.state[1])) + '}' + return ret + +def output_c_literal(hasher): + return '{0x' + hasher.digest().hex('_').replace('_', ', 0x') + '}' + +MESSAGE = b'abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmno' +assert(len(MESSAGE) == 64) +BYTE_BOUNDARIES = [(2**b)//len(MESSAGE) - 1 for b in range(20, 34)] + +midstates = [] +digests = [] +hasher = sha256() +for i in range(BYTE_BOUNDARIES[-1] + 1): + if i in BYTE_BOUNDARIES: + midstates.append(midstate_c_definition(hasher)) + hasher_copy = copy(hasher) + hasher_copy.update(MESSAGE) + digests.append(output_c_literal(hasher_copy)) + hasher.update(MESSAGE) + +for x in midstates: + print(x + ',') + +for x in digests: + print(x + ',') +``` +*/ +void run_sha256_counter_tests(void) { + static const char *input = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmno"; + static const secp256k1_sha256 midstates[] = { + {{0xa2b5c8bb, 0x26c88bb3, 0x2abdc3d2, 0x9def99a3, 0xdfd21a6e, 0x41fe585b, 0x7ef2c440, 0x2b79adda}, + {0x00}, 0xfffc0}, + {{0xa0d29445, 0x9287de66, 0x76aabd71, 0x41acd765, 0x0c7528b4, 0x84e14906, 0x942faec6, 0xcc5a7b26}, + {0x00}, 0x1fffc0}, + {{0x50449526, 0xb9f1d657, 0xa0fc13e9, 0x50860f10, 0xa550c431, 0x3fbc97c1, 0x7bbb2d89, 0xdb67bac1}, + {0x00}, 0x3fffc0}, + {{0x54a6efdc, 0x46762e7b, 0x88bfe73f, 0xbbd149c7, 0x41620c43, 0x1168da7b, 0x2c5960f9, 0xeccffda6}, + {0x00}, 0x7fffc0}, + {{0x2515a8f5, 0x5faa2977, 0x3a850486, 0xac858cad, 0x7b7276ee, 0x235c0385, 0xc53a157c, 0x7cb3e69c}, + {0x00}, 0xffffc0}, + {{0x34f39828, 0x409fedb7, 0x4bbdd0fb, 0x3b643634, 0x7806bf2e, 0xe0d1b713, 0xca3f2e1e, 0xe38722c2}, + {0x00}, 0x1ffffc0}, + {{0x389ef5c5, 0x38c54167, 0x8f5d56ab, 0x582a75cc, 0x8217caef, 0xf10947dd, 0x6a1998a8, 0x048f0b8c}, + {0x00}, 0x3ffffc0}, + {{0xd6c3f394, 0x0bee43b9, 0x6783f497, 0x29fa9e21, 0x6ce491c1, 0xa81fe45e, 0x2fc3859a, 0x269012d0}, + {0x00}, 0x7ffffc0}, + {{0x6dd3c526, 0x44d88aa0, 0x806a1bae, 0xfbcc0d32, 0x9d6144f3, 0x9d2bd757, 0x9851a957, 0xb50430ad}, + {0x00}, 0xfffffc0}, + {{0x2add4021, 0xdfe8a9e6, 0xa56317c6, 0x7a15f5bb, 0x4a48aacd, 0x5d368414, 0x4f00e6f0, 0xd9355023}, + {0x00}, 0x1fffffc0}, + {{0xb66666b4, 0xdbeac32b, 0x0ea351ae, 0xcba9da46, 0x6278b874, 0x8c508e23, 0xe16ca776, 0x8465bac1}, + {0x00}, 0x3fffffc0}, + {{0xb6744789, 0x9cce87aa, 0xc4c478b7, 0xf38404d8, 0x2e38ba62, 0xa3f7019b, 0x50458fe7, 0x3047dbec}, + {0x00}, 0x7fffffc0}, + {{0x8b1297ba, 0xba261a80, 0x2ba1b0dd, 0xfbc67d6d, 0x61072c4e, 0x4b5a2a0f, 0x52872760, 0x2dfeb162}, + {0x00}, 0xffffffc0}, + {{0x24f33cf7, 0x41ad6583, 0x41c8ff5d, 0xca7ef35f, 0x50395756, 0x021b743e, 0xd7126cd7, 0xd037473a}, + {0x00}, 0x1ffffffc0}, + }; + static const unsigned char outputs[][32] = { + {0x0e, 0x83, 0xe2, 0xc9, 0x4f, 0xb2, 0xb8, 0x2b, 0x89, 0x06, 0x92, 0x78, 0x04, 0x03, 0x48, 0x5c, 0x48, 0x44, 0x67, 0x61, 0x77, 0xa4, 0xc7, 0x90, 0x9e, 0x92, 0x55, 0x10, 0x05, 0xfe, 0x39, 0x15}, + {0x1d, 0x1e, 0xd7, 0xb8, 0xa3, 0xa7, 0x8a, 0x79, 0xfd, 0xa0, 0x05, 0x08, 0x9c, 0xeb, 0xf0, 0xec, 0x67, 0x07, 0x9f, 0x8e, 0x3c, 0x0d, 0x8e, 0xf9, 0x75, 0x55, 0x13, 0xc1, 0xe8, 0x77, 0xf8, 0xbb}, + {0x66, 0x95, 0x6c, 0xc9, 0xe0, 0x39, 0x65, 0xb6, 0xb0, 0x05, 0xd1, 0xaf, 0xaf, 0xf3, 0x1d, 0xb9, 0xa4, 0xda, 0x6f, 0x20, 0xcd, 0x3a, 0xae, 0x64, 0xc2, 0xdb, 0xee, 0xf5, 0xb8, 0x8d, 0x57, 0x0e}, + {0x3c, 0xbb, 0x1c, 0x12, 0x5e, 0x17, 0xfd, 0x54, 0x90, 0x45, 0xa7, 0x7b, 0x61, 0x6c, 0x1d, 0xfe, 0xe6, 0xcc, 0x7f, 0xee, 0xcf, 0xef, 0x33, 0x35, 0x50, 0x62, 0x16, 0x70, 0x2f, 0x87, 0xc3, 0xc9}, + {0x53, 0x4d, 0xa8, 0xe7, 0x1e, 0x98, 0x73, 0x8d, 0xd9, 0xa3, 0x54, 0xa5, 0x0e, 0x59, 0x2c, 0x25, 0x43, 0x6f, 0xaa, 0xa2, 0xf5, 0x21, 0x06, 0x3e, 0xc9, 0x82, 0x06, 0x94, 0x98, 0x72, 0x9d, 0xa7}, + {0xef, 0x7e, 0xe9, 0x6b, 0xd3, 0xe5, 0xb7, 0x41, 0x4c, 0xc8, 0xd3, 0x07, 0x52, 0x9a, 0x5a, 0x8b, 0x4e, 0x1e, 0x75, 0xa4, 0x17, 0x78, 0xc8, 0x36, 0xcd, 0xf8, 0x2e, 0xd9, 0x57, 0xe3, 0xd7, 0x07}, + {0x87, 0x16, 0xfb, 0xf9, 0xa5, 0xf8, 0xc4, 0x56, 0x2b, 0x48, 0x52, 0x8e, 0x2d, 0x30, 0x85, 0xb6, 0x4c, 0x56, 0xb5, 0xd1, 0x16, 0x9c, 0xcf, 0x32, 0x95, 0xad, 0x03, 0xe8, 0x05, 0x58, 0x06, 0x76}, + {0x75, 0x03, 0x80, 0x28, 0xf2, 0xa7, 0x63, 0x22, 0x1a, 0x26, 0x9c, 0x68, 0xe0, 0x58, 0xfc, 0x73, 0xeb, 0x42, 0xf6, 0x86, 0x16, 0x24, 0x4b, 0xbc, 0x24, 0xf7, 0x02, 0xc8, 0x3d, 0x90, 0xe2, 0xb0}, + {0xdf, 0x49, 0x0f, 0x15, 0x7b, 0x7d, 0xbf, 0xe0, 0xd4, 0xcf, 0x47, 0xc0, 0x80, 0x93, 0x4a, 0x61, 0xaa, 0x03, 0x07, 0x66, 0xb3, 0x38, 0x5d, 0xc8, 0xc9, 0x07, 0x61, 0xfb, 0x97, 0x10, 0x2f, 0xd8}, + {0x77, 0x19, 0x40, 0x56, 0x41, 0xad, 0xbc, 0x59, 0xda, 0x1e, 0xc5, 0x37, 0x14, 0x63, 0x7b, 0xfb, 0x79, 0xe2, 0x7a, 0xb1, 0x55, 0x42, 0x99, 0x42, 0x56, 0xfe, 0x26, 0x9d, 0x0f, 0x7e, 0x80, 0xc6}, + {0x50, 0xe7, 0x2a, 0x0e, 0x26, 0x44, 0x2f, 0xe2, 0x55, 0x2d, 0xc3, 0x93, 0x8a, 0xc5, 0x86, 0x58, 0x22, 0x8c, 0x0c, 0xbf, 0xb1, 0xd2, 0xca, 0x87, 0x2a, 0xe4, 0x35, 0x26, 0x6f, 0xcd, 0x05, 0x5e}, + {0xe4, 0x80, 0x6f, 0xdb, 0x3d, 0x7d, 0xba, 0xde, 0x50, 0x3f, 0xea, 0x00, 0x3d, 0x46, 0x59, 0x64, 0xfd, 0x58, 0x1c, 0xa1, 0xb8, 0x7d, 0x5f, 0xac, 0x94, 0x37, 0x9e, 0xa0, 0xc0, 0x9c, 0x93, 0x8b}, + {0x2c, 0xf3, 0xa9, 0xf6, 0x15, 0x25, 0x80, 0x70, 0x76, 0x99, 0x7d, 0xf1, 0xc3, 0x2f, 0xa3, 0x31, 0xff, 0x92, 0x35, 0x2e, 0x8d, 0x04, 0x13, 0x33, 0xd8, 0x0d, 0xdb, 0x4a, 0xf6, 0x8c, 0x03, 0x34}, + {0xec, 0x12, 0x24, 0x9f, 0x35, 0xa4, 0x29, 0x8b, 0x9e, 0x4a, 0x95, 0xf8, 0x61, 0xaf, 0x61, 0xc5, 0x66, 0x55, 0x3e, 0x3f, 0x2a, 0x98, 0xea, 0x71, 0x16, 0x6b, 0x1c, 0xd9, 0xe4, 0x09, 0xd2, 0x8e}, + }; + unsigned int i; + for (i = 0; i < sizeof(midstates)/sizeof(midstates[0]); i++) { + unsigned char out[32]; + secp256k1_sha256 hasher = midstates[i]; + secp256k1_sha256_write(&hasher, (const unsigned char*)input, strlen(input)); + secp256k1_sha256_finalize(&hasher, out); + CHECK(secp256k1_memcmp_var(out, outputs[i], 32) == 0); + } +} + void run_hmac_sha256_tests(void) { static const char *keys[6] = { "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", @@ -790,7 +921,7 @@ void signed30_to_uint16(uint16_t* out, const secp256k1_modinv32_signed30* in) { void mutate_sign_signed30(secp256k1_modinv32_signed30* x) { int i; for (i = 0; i < 16; ++i) { - int pos = secp256k1_testrand_int(8); + int pos = secp256k1_testrand_bits(3); if (x->v[pos] > 0 && x->v[pos + 1] <= 0x3fffffff) { x->v[pos] -= 0x40000000; x->v[pos + 1] += 1; @@ -862,7 +993,7 @@ void mutate_sign_signed62(secp256k1_modinv64_signed62* x) { static const int64_t M62 = (int64_t)(UINT64_MAX >> 2); int i; for (i = 0; i < 8; ++i) { - int pos = secp256k1_testrand_int(4); + int pos = secp256k1_testrand_bits(2); if (x->v[pos] > 0 && x->v[pos + 1] <= M62) { x->v[pos] -= (M62 + 1); x->v[pos + 1] += 1; @@ -2451,13 +2582,65 @@ void run_field_convert(void) { CHECK(secp256k1_memcmp_var(&fes2, &fes, sizeof(fes)) == 0); } -int fe_secp256k1_memcmp_var(const secp256k1_fe *a, const secp256k1_fe *b) { - secp256k1_fe t = *b; +/* Returns true if two field elements have the same representation. */ +int fe_identical(const secp256k1_fe *a, const secp256k1_fe *b) { + int ret = 1; #ifdef VERIFY - t.magnitude = a->magnitude; - t.normalized = a->normalized; + ret &= (a->magnitude == b->magnitude); + ret &= (a->normalized == b->normalized); #endif - return secp256k1_memcmp_var(a, &t, sizeof(secp256k1_fe)); + /* Compare the struct member that holds the limbs. */ + ret &= (secp256k1_memcmp_var(a->n, b->n, sizeof(a->n)) == 0); + return ret; +} + +void run_field_half(void) { + secp256k1_fe t, u; + int m; + + /* Check magnitude 0 input */ + secp256k1_fe_get_bounds(&t, 0); + secp256k1_fe_half(&t); +#ifdef VERIFY + CHECK(t.magnitude == 1); + CHECK(t.normalized == 0); +#endif + CHECK(secp256k1_fe_normalizes_to_zero(&t)); + + /* Check non-zero magnitudes in the supported range */ + for (m = 1; m < 32; m++) { + /* Check max-value input */ + secp256k1_fe_get_bounds(&t, m); + + u = t; + secp256k1_fe_half(&u); +#ifdef VERIFY + CHECK(u.magnitude == (m >> 1) + 1); + CHECK(u.normalized == 0); +#endif + secp256k1_fe_normalize_weak(&u); + secp256k1_fe_add(&u, &u); + CHECK(check_fe_equal(&t, &u)); + + /* Check worst-case input: ensure the LSB is 1 so that P will be added, + * which will also cause all carries to be 1, since all limbs that can + * generate a carry are initially even and all limbs of P are odd in + * every existing field implementation. */ + secp256k1_fe_get_bounds(&t, m); + CHECK(t.n[0] > 0); + CHECK((t.n[0] & 1) == 0); + --t.n[0]; + + u = t; + secp256k1_fe_half(&u); +#ifdef VERIFY + CHECK(u.magnitude == (m >> 1) + 1); + CHECK(u.normalized == 0); +#endif + secp256k1_fe_normalize_weak(&u); + secp256k1_fe_add(&u, &u); + CHECK(check_fe_equal(&t, &u)); + } } void run_field_misc(void) { @@ -2467,9 +2650,13 @@ void run_field_misc(void) { secp256k1_fe q; secp256k1_fe fe5 = SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 5); int i, j; - for (i = 0; i < 5*count; i++) { + for (i = 0; i < 1000 * count; i++) { secp256k1_fe_storage xs, ys, zs; - random_fe(&x); + if (i & 1) { + random_fe(&x); + } else { + random_fe_test(&x); + } random_fe_non_zero(&y); /* Test the fe equality and comparison operations. */ CHECK(secp256k1_fe_cmp_var(&x, &x) == 0); @@ -2483,13 +2670,13 @@ void run_field_misc(void) { CHECK(x.normalized && x.magnitude == 1); #endif secp256k1_fe_cmov(&x, &x, 1); - CHECK(fe_secp256k1_memcmp_var(&x, &z) != 0); - CHECK(fe_secp256k1_memcmp_var(&x, &q) == 0); + CHECK(!fe_identical(&x, &z)); + CHECK(fe_identical(&x, &q)); secp256k1_fe_cmov(&q, &z, 1); #ifdef VERIFY CHECK(!q.normalized && q.magnitude == z.magnitude); #endif - CHECK(fe_secp256k1_memcmp_var(&q, &z) == 0); + CHECK(fe_identical(&q, &z)); secp256k1_fe_normalize_var(&x); secp256k1_fe_normalize_var(&z); CHECK(!secp256k1_fe_equal_var(&x, &z)); @@ -2537,6 +2724,14 @@ void run_field_misc(void) { secp256k1_fe_add(&q, &x); CHECK(check_fe_equal(&y, &z)); CHECK(check_fe_equal(&q, &y)); + /* Check secp256k1_fe_half. */ + z = x; + secp256k1_fe_half(&z); + secp256k1_fe_add(&z, &z); + CHECK(check_fe_equal(&x, &z)); + secp256k1_fe_add(&z, &z); + secp256k1_fe_half(&z); + CHECK(check_fe_equal(&x, &z)); } } @@ -3338,6 +3533,37 @@ void run_ge(void) { test_intialized_inf(); } +void test_gej_cmov(const secp256k1_gej *a, const secp256k1_gej *b) { + secp256k1_gej t = *a; + secp256k1_gej_cmov(&t, b, 0); + CHECK(gej_xyz_equals_gej(&t, a)); + secp256k1_gej_cmov(&t, b, 1); + CHECK(gej_xyz_equals_gej(&t, b)); +} + +void run_gej(void) { + int i; + secp256k1_gej a, b; + + /* Tests for secp256k1_gej_cmov */ + for (i = 0; i < count; i++) { + secp256k1_gej_set_infinity(&a); + secp256k1_gej_set_infinity(&b); + test_gej_cmov(&a, &b); + + random_gej_test(&a); + test_gej_cmov(&a, &b); + test_gej_cmov(&b, &a); + + b = a; + test_gej_cmov(&a, &b); + + random_gej_test(&b); + test_gej_cmov(&a, &b); + test_gej_cmov(&b, &a); + } +} + void test_ec_combine(void) { secp256k1_scalar sum = SECP256K1_SCALAR_CONST(0, 0, 0, 0, 0, 0, 0, 0); secp256k1_pubkey data[6]; @@ -4052,6 +4278,174 @@ void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi_func e } } +int test_ecmult_multi_random(secp256k1_scratch *scratch) { + /* Large random test for ecmult_multi_* functions which exercises: + * - Few or many inputs (0 up to 128, roughly exponentially distributed). + * - Few or many 0*P or a*INF inputs (roughly uniformly distributed). + * - Including or excluding an nonzero a*G term (or such a term at all). + * - Final expected result equal to infinity or not (roughly 50%). + * - ecmult_multi_var, ecmult_strauss_single_batch, ecmult_pippenger_single_batch + */ + + /* These 4 variables define the eventual input to the ecmult_multi function. + * g_scalar is the G scalar fed to it (or NULL, possibly, if g_scalar=0), and + * scalars[0..filled-1] and gejs[0..filled-1] are the scalars and points + * which form its normal inputs. */ + int filled = 0; + secp256k1_scalar g_scalar = SECP256K1_SCALAR_CONST(0, 0, 0, 0, 0, 0, 0, 0); + secp256k1_scalar scalars[128]; + secp256k1_gej gejs[128]; + /* The expected result, and the computed result. */ + secp256k1_gej expected, computed; + /* Temporaries. */ + secp256k1_scalar sc_tmp; + secp256k1_ge ge_tmp; + /* Variables needed for the actual input to ecmult_multi. */ + secp256k1_ge ges[128]; + ecmult_multi_data data; + + int i; + /* Which multiplication function to use */ + int fn = secp256k1_testrand_int(3); + secp256k1_ecmult_multi_func ecmult_multi = fn == 0 ? secp256k1_ecmult_multi_var : + fn == 1 ? secp256k1_ecmult_strauss_batch_single : + secp256k1_ecmult_pippenger_batch_single; + /* Simulate exponentially distributed num. */ + int num_bits = 2 + secp256k1_testrand_int(6); + /* Number of (scalar, point) inputs (excluding g). */ + int num = secp256k1_testrand_int((1 << num_bits) + 1); + /* Number of those which are nonzero. */ + int num_nonzero = secp256k1_testrand_int(num + 1); + /* Whether we're aiming to create an input with nonzero expected result. */ + int nonzero_result = secp256k1_testrand_bits(1); + /* Whether we will provide nonzero g multiplicand. In some cases our hand + * is forced here based on num_nonzero and nonzero_result. */ + int g_nonzero = num_nonzero == 0 ? nonzero_result : + num_nonzero == 1 && !nonzero_result ? 1 : + (int)secp256k1_testrand_bits(1); + /* Which g_scalar pointer to pass into ecmult_multi(). */ + const secp256k1_scalar* g_scalar_ptr = (g_nonzero || secp256k1_testrand_bits(1)) ? &g_scalar : NULL; + /* How many EC multiplications were performed in this function. */ + int mults = 0; + /* How many randomization steps to apply to the input list. */ + int rands = (int)secp256k1_testrand_bits(3); + if (rands > num_nonzero) rands = num_nonzero; + + secp256k1_gej_set_infinity(&expected); + secp256k1_gej_set_infinity(&gejs[0]); + secp256k1_scalar_set_int(&scalars[0], 0); + + if (g_nonzero) { + /* If g_nonzero, set g_scalar to nonzero value r. */ + random_scalar_order_test(&g_scalar); + if (!nonzero_result) { + /* If expected=0 is desired, add a (a*r, -(1/a)*g) term to compensate. */ + CHECK(num_nonzero > filled); + random_scalar_order_test(&sc_tmp); + secp256k1_scalar_mul(&scalars[filled], &sc_tmp, &g_scalar); + secp256k1_scalar_inverse_var(&sc_tmp, &sc_tmp); + secp256k1_scalar_negate(&sc_tmp, &sc_tmp); + secp256k1_ecmult_gen(&ctx->ecmult_gen_ctx, &gejs[filled], &sc_tmp); + ++filled; + ++mults; + } + } + + if (nonzero_result && filled < num_nonzero) { + /* If a nonzero result is desired, and there is space, add a random nonzero term. */ + random_scalar_order_test(&scalars[filled]); + random_group_element_test(&ge_tmp); + secp256k1_gej_set_ge(&gejs[filled], &ge_tmp); + ++filled; + } + + if (nonzero_result) { + /* Compute the expected result using normal ecmult. */ + CHECK(filled <= 1); + secp256k1_ecmult(&expected, &gejs[0], &scalars[0], &g_scalar); + mults += filled + g_nonzero; + } + + /* At this point we have expected = scalar_g*G + sum(scalars[i]*gejs[i] for i=0..filled-1). */ + CHECK(filled <= 1 + !nonzero_result); + CHECK(filled <= num_nonzero); + + /* Add entries to scalars,gejs so that there are num of them. All the added entries + * either have scalar=0 or point=infinity, so these do not change the expected result. */ + while (filled < num) { + if (secp256k1_testrand_bits(1)) { + secp256k1_gej_set_infinity(&gejs[filled]); + random_scalar_order_test(&scalars[filled]); + } else { + secp256k1_scalar_set_int(&scalars[filled], 0); + random_group_element_test(&ge_tmp); + secp256k1_gej_set_ge(&gejs[filled], &ge_tmp); + } + ++filled; + } + + /* Now perform cheapish transformations on gejs and scalars, for indices + * 0..num_nonzero-1, which do not change the expected result, but may + * convert some of them to be both non-0-scalar and non-infinity-point. */ + for (i = 0; i < rands; ++i) { + int j; + secp256k1_scalar v, iv; + /* Shuffle the entries. */ + for (j = 0; j < num_nonzero; ++j) { + int k = secp256k1_testrand_int(num_nonzero - j); + if (k != 0) { + secp256k1_gej gej = gejs[j]; + secp256k1_scalar sc = scalars[j]; + gejs[j] = gejs[j + k]; + scalars[j] = scalars[j + k]; + gejs[j + k] = gej; + scalars[j + k] = sc; + } + } + /* Perturb all consecutive pairs of inputs: + * a*P + b*Q -> (a+b)*P + b*(Q-P). */ + for (j = 0; j + 1 < num_nonzero; j += 2) { + secp256k1_gej gej; + secp256k1_scalar_add(&scalars[j], &scalars[j], &scalars[j+1]); + secp256k1_gej_neg(&gej, &gejs[j]); + secp256k1_gej_add_var(&gejs[j+1], &gejs[j+1], &gej, NULL); + } + /* Transform the last input: a*P -> (v*a) * ((1/v)*P). */ + CHECK(num_nonzero >= 1); + random_scalar_order_test(&v); + secp256k1_scalar_inverse(&iv, &v); + secp256k1_scalar_mul(&scalars[num_nonzero - 1], &scalars[num_nonzero - 1], &v); + secp256k1_ecmult(&gejs[num_nonzero - 1], &gejs[num_nonzero - 1], &iv, NULL); + ++mults; + } + + /* Shuffle all entries (0..num-1). */ + for (i = 0; i < num; ++i) { + int j = secp256k1_testrand_int(num - i); + if (j != 0) { + secp256k1_gej gej = gejs[i]; + secp256k1_scalar sc = scalars[i]; + gejs[i] = gejs[i + j]; + scalars[i] = scalars[i + j]; + gejs[i + j] = gej; + scalars[i + j] = sc; + } + } + + /* Compute affine versions of all inputs. */ + secp256k1_ge_set_all_gej_var(ges, gejs, filled); + /* Invoke ecmult_multi code. */ + data.sc = scalars; + data.pt = ges; + CHECK(ecmult_multi(&ctx->error_callback, scratch, &computed, g_scalar_ptr, ecmult_multi_callback, &data, filled)); + mults += num_nonzero + g_nonzero; + /* Compare with expected result. */ + secp256k1_gej_neg(&computed, &computed); + secp256k1_gej_add_var(&computed, &computed, &expected, NULL); + CHECK(secp256k1_gej_is_infinity(&computed)); + return mults; +} + void test_ecmult_multi_batch_single(secp256k1_ecmult_multi_func ecmult_multi) { secp256k1_scalar szero; secp256k1_scalar sc; @@ -4093,7 +4487,7 @@ void test_secp256k1_pippenger_bucket_window_inv(void) { * for a given scratch space. */ void test_ecmult_multi_pippenger_max_points(void) { - size_t scratch_size = secp256k1_testrand_int(256); + size_t scratch_size = secp256k1_testrand_bits(8); size_t max_size = secp256k1_pippenger_scratch_size(secp256k1_pippenger_bucket_window_inv(PIPPENGER_MAX_BUCKET_WINDOW-1)+512, 12); secp256k1_scratch *scratch; size_t n_points_supported; @@ -4242,6 +4636,7 @@ void test_ecmult_multi_batching(void) { void run_ecmult_multi_tests(void) { secp256k1_scratch *scratch; + int64_t todo = (int64_t)320 * count; test_secp256k1_pippenger_bucket_window_inv(); test_ecmult_multi_pippenger_max_points(); @@ -4252,6 +4647,9 @@ void run_ecmult_multi_tests(void) { test_ecmult_multi_batch_single(secp256k1_ecmult_pippenger_batch_single); test_ecmult_multi(scratch, secp256k1_ecmult_strauss_batch_single); test_ecmult_multi_batch_single(secp256k1_ecmult_strauss_batch_single); + while (todo > 0) { + todo -= test_ecmult_multi_random(scratch); + } secp256k1_scratch_destroy(&ctx->error_callback, scratch); /* Run test_ecmult_multi with space for exactly one point */ @@ -4347,7 +4745,7 @@ void test_constant_wnaf(const secp256k1_scalar *number, int w) { secp256k1_scalar_add(&x, &x, &t); } /* Skew num because when encoding numbers as odd we use an offset */ - secp256k1_scalar_set_int(&scalar_skew, 1 << (skew == 2)); + secp256k1_scalar_set_int(&scalar_skew, skew); secp256k1_scalar_add(&num, &num, &scalar_skew); CHECK(secp256k1_scalar_eq(&x, &num)); } @@ -4540,8 +4938,8 @@ void test_ecmult_accumulate(secp256k1_sha256* acc, const secp256k1_scalar* x, se } } -void test_ecmult_constants(void) { - /* Test ecmult_gen for: +void test_ecmult_constants_2bit(void) { + /* Using test_ecmult_accumulate, test ecmult for: * - For i in 0..36: * - Key i * - Key -i @@ -4584,8 +4982,81 @@ void test_ecmult_constants(void) { secp256k1_scratch_space_destroy(ctx, scratch); } +void test_ecmult_constants_sha(uint32_t prefix, size_t iter, const unsigned char* expected32) { + /* Using test_ecmult_accumulate, test ecmult for: + * - Key 0 + * - Key 1 + * - Key -1 + * - For i in range(iter): + * - Key SHA256(LE32(prefix) || LE16(i)) + */ + secp256k1_scalar x; + secp256k1_sha256 acc; + unsigned char b32[32]; + unsigned char inp[6]; + size_t i; + secp256k1_scratch_space *scratch = secp256k1_scratch_space_create(ctx, 65536); + + inp[0] = prefix & 0xFF; + inp[1] = (prefix >> 8) & 0xFF; + inp[2] = (prefix >> 16) & 0xFF; + inp[3] = (prefix >> 24) & 0xFF; + secp256k1_sha256_initialize(&acc); + secp256k1_scalar_set_int(&x, 0); + test_ecmult_accumulate(&acc, &x, scratch); + secp256k1_scalar_set_int(&x, 1); + test_ecmult_accumulate(&acc, &x, scratch); + secp256k1_scalar_negate(&x, &x); + test_ecmult_accumulate(&acc, &x, scratch); + + for (i = 0; i < iter; ++i) { + secp256k1_sha256 gen; + inp[4] = i & 0xff; + inp[5] = (i >> 8) & 0xff; + secp256k1_sha256_initialize(&gen); + secp256k1_sha256_write(&gen, inp, sizeof(inp)); + secp256k1_sha256_finalize(&gen, b32); + secp256k1_scalar_set_b32(&x, b32, NULL); + test_ecmult_accumulate(&acc, &x, scratch); + } + secp256k1_sha256_finalize(&acc, b32); + CHECK(secp256k1_memcmp_var(b32, expected32, 32) == 0); + + secp256k1_scratch_space_destroy(ctx, scratch); +} + void run_ecmult_constants(void) { - test_ecmult_constants(); + /* Expected hashes of all points in the tests below. Computed using an + * independent implementation. */ + static const unsigned char expected32_6bit20[32] = { + 0x68, 0xb6, 0xed, 0x6f, 0x28, 0xca, 0xc9, 0x7f, + 0x8e, 0x8b, 0xd6, 0xc0, 0x61, 0x79, 0x34, 0x6e, + 0x5a, 0x8f, 0x2b, 0xbc, 0x3e, 0x1f, 0xc5, 0x2e, + 0x2a, 0xd0, 0x45, 0x67, 0x7f, 0x95, 0x95, 0x8e + }; + static const unsigned char expected32_8bit8[32] = { + 0x8b, 0x65, 0x8e, 0xea, 0x86, 0xae, 0x3c, 0x95, + 0x90, 0xb6, 0x77, 0xa4, 0x8c, 0x76, 0xd9, 0xec, + 0xf5, 0xab, 0x8a, 0x2f, 0xfd, 0xdb, 0x19, 0x12, + 0x1a, 0xee, 0xe6, 0xb7, 0x6e, 0x05, 0x3f, 0xc6 + }; + /* For every combination of 6 bit positions out of 256, restricted to + * 20-bit windows (i.e., the first and last bit position are no more than + * 19 bits apart), all 64 bit patterns occur in the input scalars used in + * this test. */ + CONDITIONAL_TEST(1, "test_ecmult_constants_sha 1024") { + test_ecmult_constants_sha(4808378u, 1024, expected32_6bit20); + } + + /* For every combination of 8 consecutive bit positions, all 256 bit + * patterns occur in the input scalars used in this test. */ + CONDITIONAL_TEST(3, "test_ecmult_constants_sha 2048") { + test_ecmult_constants_sha(1607366309u, 2048, expected32_8bit8); + } + + CONDITIONAL_TEST(35, "test_ecmult_constants_2bit") { + test_ecmult_constants_2bit(); + } } void test_ecmult_gen_blind(void) { @@ -5851,14 +6322,14 @@ static void random_ber_signature(unsigned char *sig, size_t *len, int* certainly /* We generate two classes of numbers: nlow==1 "low" ones (up to 32 bytes), nlow==0 "high" ones (32 bytes with 129 top bits set, or larger than 32 bytes) */ nlow[n] = der ? 1 : (secp256k1_testrand_bits(3) != 0); /* The length of the number in bytes (the first byte of which will always be nonzero) */ - nlen[n] = nlow[n] ? secp256k1_testrand_int(33) : 32 + secp256k1_testrand_int(200) * secp256k1_testrand_int(8) / 8; + nlen[n] = nlow[n] ? secp256k1_testrand_int(33) : 32 + secp256k1_testrand_int(200) * secp256k1_testrand_bits(3) / 8; CHECK(nlen[n] <= 232); /* The top bit of the number. */ nhbit[n] = (nlow[n] == 0 && nlen[n] == 32) ? 1 : (nlen[n] == 0 ? 0 : secp256k1_testrand_bits(1)); /* The top byte of the number (after the potential hardcoded 16 0xFF characters for "high" 32 bytes numbers) */ nhbyte[n] = nlen[n] == 0 ? 0 : (nhbit[n] ? 128 + secp256k1_testrand_bits(7) : 1 + secp256k1_testrand_int(127)); /* The number of zero bytes in front of the number (which is 0 or 1 in case of DER, otherwise we extend up to 300 bytes) */ - nzlen[n] = der ? ((nlen[n] == 0 || nhbit[n]) ? 1 : 0) : (nlow[n] ? secp256k1_testrand_int(3) : secp256k1_testrand_int(300 - nlen[n]) * secp256k1_testrand_int(8) / 8); + nzlen[n] = der ? ((nlen[n] == 0 || nhbit[n]) ? 1 : 0) : (nlow[n] ? secp256k1_testrand_int(3) : secp256k1_testrand_int(300 - nlen[n]) * secp256k1_testrand_bits(3) / 8); if (nzlen[n] > ((nlen[n] == 0 || nhbit[n]) ? 1 : 0)) { *certainly_not_der = 1; } @@ -5867,7 +6338,7 @@ static void random_ber_signature(unsigned char *sig, size_t *len, int* certainly nlenlen[n] = nlen[n] + nzlen[n] < 128 ? 0 : (nlen[n] + nzlen[n] < 256 ? 1 : 2); if (!der) { /* nlenlen[n] max 127 bytes */ - int add = secp256k1_testrand_int(127 - nlenlen[n]) * secp256k1_testrand_int(16) * secp256k1_testrand_int(16) / 256; + int add = secp256k1_testrand_int(127 - nlenlen[n]) * secp256k1_testrand_bits(4) * secp256k1_testrand_bits(4) / 256; nlenlen[n] += add; if (add != 0) { *certainly_not_der = 1; @@ -5881,7 +6352,7 @@ static void random_ber_signature(unsigned char *sig, size_t *len, int* certainly CHECK(tlen <= 856); /* The length of the garbage inside the tuple. */ - elen = (der || indet) ? 0 : secp256k1_testrand_int(980 - tlen) * secp256k1_testrand_int(8) / 8; + elen = (der || indet) ? 0 : secp256k1_testrand_int(980 - tlen) * secp256k1_testrand_bits(3) / 8; if (elen != 0) { *certainly_not_der = 1; } @@ -5889,7 +6360,7 @@ static void random_ber_signature(unsigned char *sig, size_t *len, int* certainly CHECK(tlen <= 980); /* The length of the garbage after the end of the tuple. */ - glen = der ? 0 : secp256k1_testrand_int(990 - tlen) * secp256k1_testrand_int(8) / 8; + glen = der ? 0 : secp256k1_testrand_int(990 - tlen) * secp256k1_testrand_bits(3) / 8; if (glen != 0) { *certainly_not_der = 1; } @@ -5904,7 +6375,7 @@ static void random_ber_signature(unsigned char *sig, size_t *len, int* certainly } else { int tlenlen = tlen < 128 ? 0 : (tlen < 256 ? 1 : 2); if (!der) { - int add = secp256k1_testrand_int(127 - tlenlen) * secp256k1_testrand_int(16) * secp256k1_testrand_int(16) / 256; + int add = secp256k1_testrand_int(127 - tlenlen) * secp256k1_testrand_bits(4) * secp256k1_testrand_bits(4) / 256; tlenlen += add; if (add != 0) { *certainly_not_der = 1; @@ -6416,6 +6887,19 @@ void run_secp256k1_memczero_test(void) { CHECK(secp256k1_memcmp_var(buf1, buf2, sizeof(buf1)) == 0); } +void run_secp256k1_byteorder_tests(void) { + const uint32_t x = 0xFF03AB45; + const unsigned char x_be[4] = {0xFF, 0x03, 0xAB, 0x45}; + unsigned char buf[4]; + uint32_t x_; + + secp256k1_write_be32(buf, x); + CHECK(secp256k1_memcmp_var(buf, x_be, sizeof(buf)) == 0); + + x_ = secp256k1_read_be32(buf); + CHECK(x == x_); +} + void int_cmov_test(void) { int r = INT_MAX; int a = 0; @@ -6616,7 +7100,8 @@ int main(int argc, char **argv) { run_modinv_tests(); run_inverse_tests(); - run_sha256_tests(); + run_sha256_known_output_tests(); + run_sha256_counter_tests(); run_hmac_sha256_tests(); run_rfc6979_hmac_sha256_tests(); run_tagged_sha256_tests(); @@ -6625,6 +7110,7 @@ int main(int argc, char **argv) { run_scalar_tests(); /* field tests */ + run_field_half(); run_field_misc(); run_field_convert(); run_fe_mul(); @@ -6633,6 +7119,7 @@ int main(int argc, char **argv) { /* group tests */ run_ge(); + run_gej(); run_group_decompress(); /* ecmult tests */ @@ -6687,6 +7174,7 @@ int main(int argc, char **argv) { /* util tests */ run_secp256k1_memczero_test(); + run_secp256k1_byteorder_tests(); run_cmov_tests(); diff --git a/src/secp256k1/src/tests_exhaustive.c b/src/secp256k1/src/tests_exhaustive.c index 6bae7a4778..6a4e2340f2 100644 --- a/src/secp256k1/src/tests_exhaustive.c +++ b/src/secp256k1/src/tests_exhaustive.c @@ -22,7 +22,8 @@ #include "assumptions.h" #include "group.h" #include "testrand_impl.h" -#include "ecmult_gen_prec_impl.h" +#include "ecmult_compute_table_impl.h" +#include "ecmult_gen_compute_table_impl.h" static int count = 2; @@ -389,8 +390,9 @@ int main(int argc, char** argv) { printf("running tests for core %lu (out of [0..%lu])\n", (unsigned long)this_core, (unsigned long)num_cores - 1); } - /* Recreate the ecmult_gen table using the right generator (as selected via EXHAUSTIVE_TEST_ORDER) */ - secp256k1_ecmult_gen_create_prec_table(&secp256k1_ecmult_gen_prec_table[0][0], &secp256k1_ge_const_g, ECMULT_GEN_PREC_BITS); + /* Recreate the ecmult{,_gen} tables using the right generator (as selected via EXHAUSTIVE_TEST_ORDER) */ + secp256k1_ecmult_gen_compute_table(&secp256k1_ecmult_gen_prec_table[0][0], &secp256k1_ge_const_g, ECMULT_GEN_PREC_BITS); + secp256k1_ecmult_compute_two_tables(secp256k1_pre_g, secp256k1_pre_g_128, WINDOW_G, &secp256k1_ge_const_g); while (count--) { /* Build context */ diff --git a/src/secp256k1/src/util.h b/src/secp256k1/src/util.h index 04227a7c9b..dac86bd77f 100644 --- a/src/secp256k1/src/util.h +++ b/src/secp256k1/src/util.h @@ -173,31 +173,6 @@ static SECP256K1_INLINE void *checked_realloc(const secp256k1_callback* cb, void # define SECP256K1_GNUC_EXT #endif -/* If SECP256K1_{LITTLE,BIG}_ENDIAN is not explicitly provided, infer from various other system macros. */ -#if !defined(SECP256K1_LITTLE_ENDIAN) && !defined(SECP256K1_BIG_ENDIAN) -/* Inspired by https://github.com/rofl0r/endianness.h/blob/9853923246b065a3b52d2c43835f3819a62c7199/endianness.h#L52L73 */ -# if (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) || \ - defined(_X86_) || defined(__x86_64__) || defined(__i386__) || \ - defined(__i486__) || defined(__i586__) || defined(__i686__) || \ - defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) || \ - defined(__ARMEL__) || defined(__AARCH64EL__) || \ - (defined(__LITTLE_ENDIAN__) && __LITTLE_ENDIAN__ == 1) || \ - (defined(_LITTLE_ENDIAN) && _LITTLE_ENDIAN == 1) || \ - defined(_M_IX86) || defined(_M_AMD64) || defined(_M_ARM) /* MSVC */ -# define SECP256K1_LITTLE_ENDIAN -# endif -# if (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) || \ - defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) || \ - defined(__MICROBLAZEEB__) || defined(__ARMEB__) || defined(__AARCH64EB__) || \ - (defined(__BIG_ENDIAN__) && __BIG_ENDIAN__ == 1) || \ - (defined(_BIG_ENDIAN) && _BIG_ENDIAN == 1) -# define SECP256K1_BIG_ENDIAN -# endif -#endif -#if defined(SECP256K1_LITTLE_ENDIAN) == defined(SECP256K1_BIG_ENDIAN) -# error Please make sure that either SECP256K1_LITTLE_ENDIAN or SECP256K1_BIG_ENDIAN is set, see src/util.h. -#endif - /* Zero memory if flag == 1. Flag must be 0 or 1. Constant time. */ static SECP256K1_INLINE void secp256k1_memczero(void *s, size_t len, int flag) { unsigned char *p = (unsigned char *)s; @@ -338,4 +313,20 @@ static SECP256K1_INLINE int secp256k1_ctz64_var(uint64_t x) { #endif } +/* Read a uint32_t in big endian */ +SECP256K1_INLINE static uint32_t secp256k1_read_be32(const unsigned char* p) { + return (uint32_t)p[0] << 24 | + (uint32_t)p[1] << 16 | + (uint32_t)p[2] << 8 | + (uint32_t)p[3]; +} + +/* Write a uint32_t in big endian */ +SECP256K1_INLINE static void secp256k1_write_be32(unsigned char* p, uint32_t x) { + p[3] = x; + p[2] = x >> 8; + p[1] = x >> 16; + p[0] = x >> 24; +} + #endif /* SECP256K1_UTIL_H */ diff --git a/src/secp256k1/src/valgrind_ctime_test.c b/src/secp256k1/src/valgrind_ctime_test.c index ea6d4b3deb..6ff0085d34 100644 --- a/src/secp256k1/src/valgrind_ctime_test.c +++ b/src/secp256k1/src/valgrind_ctime_test.c @@ -166,7 +166,7 @@ void run_tests(secp256k1_context *ctx, unsigned char *key) { ret = secp256k1_keypair_create(ctx, &keypair, key); VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); CHECK(ret == 1); - ret = secp256k1_schnorrsig_sign(ctx, sig, msg, &keypair, NULL); + ret = secp256k1_schnorrsig_sign32(ctx, sig, msg, &keypair, NULL); VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); CHECK(ret == 1); #endif diff --git a/src/serialize.h b/src/serialize.h index 44bb471f25..a1cce78451 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -472,10 +472,10 @@ struct CustomUintFormatter if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range"); if (BigEndian) { uint64_t raw = htobe64(v); - s.write({BytePtr(&raw) + 8 - Bytes, Bytes}); + s.write({AsBytePtr(&raw) + 8 - Bytes, Bytes}); } else { uint64_t raw = htole64(v); - s.write({BytePtr(&raw), Bytes}); + s.write({AsBytePtr(&raw), Bytes}); } } @@ -485,10 +485,10 @@ struct CustomUintFormatter static_assert(std::numeric_limits<U>::max() >= MAX && std::numeric_limits<U>::min() <= 0, "Assigned type too small"); uint64_t raw = 0; if (BigEndian) { - s.read({BytePtr(&raw) + 8 - Bytes, Bytes}); + s.read({AsBytePtr(&raw) + 8 - Bytes, Bytes}); v = static_cast<I>(be64toh(raw)); } else { - s.read({BytePtr(&raw), Bytes}); + s.read({AsBytePtr(&raw), Bytes}); v = static_cast<I>(le64toh(raw)); } } diff --git a/src/shutdown.cpp b/src/shutdown.cpp index 93f04e9021..fdf726b5f1 100644 --- a/src/shutdown.cpp +++ b/src/shutdown.cpp @@ -5,13 +5,15 @@ #include <shutdown.h> +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + #include <logging.h> #include <node/ui_interface.h> #include <util/tokenpipe.h> #include <warnings.h> -#include <config/bitcoin-config.h> - #include <assert.h> #include <atomic> #ifdef WIN32 diff --git a/src/span.h b/src/span.h index b627b993c2..444d63001f 100644 --- a/src/span.h +++ b/src/span.h @@ -245,19 +245,19 @@ T& SpanPopBack(Span<T>& span) //! Convert a data pointer to a std::byte data pointer. //! Where possible, please use the safer AsBytes helpers. -inline const std::byte* BytePtr(const void* data) { return reinterpret_cast<const std::byte*>(data); } -inline std::byte* BytePtr(void* data) { return reinterpret_cast<std::byte*>(data); } +inline const std::byte* AsBytePtr(const void* data) { return reinterpret_cast<const std::byte*>(data); } +inline std::byte* AsBytePtr(void* data) { return reinterpret_cast<std::byte*>(data); } // From C++20 as_bytes and as_writeable_bytes template <typename T> Span<const std::byte> AsBytes(Span<T> s) noexcept { - return {BytePtr(s.data()), s.size_bytes()}; + return {AsBytePtr(s.data()), s.size_bytes()}; } template <typename T> Span<std::byte> AsWritableBytes(Span<T> s) noexcept { - return {BytePtr(s.data()), s.size_bytes()}; + return {AsBytePtr(s.data()), s.size_bytes()}; } template <typename V> diff --git a/src/support/allocators/secure.h b/src/support/allocators/secure.h index a9aa928082..c6bd685189 100644 --- a/src/support/allocators/secure.h +++ b/src/support/allocators/secure.h @@ -37,7 +37,7 @@ struct secure_allocator : public std::allocator<T> { typedef secure_allocator<_Other> other; }; - T* allocate(std::size_t n, const void* hint = 0) + T* allocate(std::size_t n, const void* hint = nullptr) { T* allocation = static_cast<T*>(LockedPoolManager::Instance().alloc(sizeof(T) * n)); if (!allocation) { diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp index 6965f40253..ea1a27c6f6 100644 --- a/src/support/lockedpool.cpp +++ b/src/support/lockedpool.cpp @@ -235,12 +235,6 @@ PosixLockedPageAllocator::PosixLockedPageAllocator() #endif } -// Some systems (at least OS X) do not define MAP_ANONYMOUS yet and define -// MAP_ANON which is deprecated -#ifndef MAP_ANONYMOUS -#define MAP_ANONYMOUS MAP_ANON -#endif - void *PosixLockedPageAllocator::AllocateLocked(size_t len, bool *lockingSuccess) { void *addr; diff --git a/src/sync.h b/src/sync.h index 85000a9151..c69b58741b 100644 --- a/src/sync.h +++ b/src/sync.h @@ -6,8 +6,11 @@ #ifndef BITCOIN_SYNC_H #define BITCOIN_SYNC_H +#ifdef DEBUG_LOCKCONTENTION #include <logging.h> #include <logging/timer.h> +#endif + #include <threadsafety.h> #include <util/macros.h> @@ -136,8 +139,10 @@ private: void Enter(const char* pszName, const char* pszFile, int nLine) { EnterCritical(pszName, pszFile, nLine, Base::mutex()); +#ifdef DEBUG_LOCKCONTENTION if (Base::try_lock()) return; LOG_TIME_MICROS_WITH_CATEGORY(strprintf("lock contention %s, %s:%d", pszName, pszFile, nLine), BCLog::LOCK); +#endif Base::lock(); } @@ -218,12 +223,12 @@ public: friend class reverse_lock; }; -#define REVERSE_LOCK(g) typename std::decay<decltype(g)>::type::reverse_lock PASTE2(revlock, __COUNTER__)(g, #g, __FILE__, __LINE__) +#define REVERSE_LOCK(g) typename std::decay<decltype(g)>::type::reverse_lock UNIQUE_NAME(revlock)(g, #g, __FILE__, __LINE__) template<typename MutexArg> using DebugLock = UniqueLock<typename std::remove_reference<typename std::remove_pointer<MutexArg>::type>::type>; -#define LOCK(cs) DebugLock<decltype(cs)> PASTE2(criticalblock, __COUNTER__)(cs, #cs, __FILE__, __LINE__) +#define LOCK(cs) DebugLock<decltype(cs)> UNIQUE_NAME(criticalblock)(cs, #cs, __FILE__, __LINE__) #define LOCK2(cs1, cs2) \ DebugLock<decltype(cs1)> criticalblock1(cs1, #cs1, __FILE__, __LINE__); \ DebugLock<decltype(cs2)> criticalblock2(cs2, #cs2, __FILE__, __LINE__); diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index efc30b6822..12cf1176a6 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -23,7 +23,7 @@ using namespace std::literals; using node::NodeContext; -static const std::vector<bool> EMPTY_ASMAP; +static NetGroupManager EMPTY_NETGROUPMAN{std::vector<bool>()}; static const bool DETERMINISTIC{true}; static int32_t GetCheckRatio(const NodeContext& node_ctx) @@ -62,7 +62,7 @@ BOOST_FIXTURE_TEST_SUITE(addrman_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(addrman_simple) { - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); CNetAddr source = ResolveIP("252.2.2.2"); @@ -96,7 +96,7 @@ BOOST_AUTO_TEST_CASE(addrman_simple) BOOST_CHECK(addrman->size() >= 1); // Test: reset addrman and test AddrMan::Add multiple addresses works as expected - addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); std::vector<CAddress> vAddr; vAddr.push_back(CAddress(ResolveService("250.1.1.3", 8333), NODE_NONE)); vAddr.push_back(CAddress(ResolveService("250.1.1.4", 8333), NODE_NONE)); @@ -106,7 +106,7 @@ BOOST_AUTO_TEST_CASE(addrman_simple) BOOST_AUTO_TEST_CASE(addrman_ports) { - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); CNetAddr source = ResolveIP("252.2.2.2"); @@ -135,7 +135,7 @@ BOOST_AUTO_TEST_CASE(addrman_ports) BOOST_AUTO_TEST_CASE(addrman_select) { - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); CNetAddr source = ResolveIP("252.2.2.2"); @@ -194,7 +194,7 @@ BOOST_AUTO_TEST_CASE(addrman_select) BOOST_AUTO_TEST_CASE(addrman_new_collisions) { - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); CNetAddr source = ResolveIP("252.2.2.2"); @@ -223,7 +223,7 @@ BOOST_AUTO_TEST_CASE(addrman_new_collisions) BOOST_AUTO_TEST_CASE(addrman_new_multiplicity) { - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); CAddress addr{CAddress(ResolveService("253.3.3.3", 8333), NODE_NONE)}; int64_t start_time{GetAdjustedTime()}; addr.nTime = start_time; @@ -255,7 +255,7 @@ BOOST_AUTO_TEST_CASE(addrman_new_multiplicity) BOOST_AUTO_TEST_CASE(addrman_tried_collisions) { - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); CNetAddr source = ResolveIP("252.2.2.2"); @@ -286,7 +286,7 @@ BOOST_AUTO_TEST_CASE(addrman_tried_collisions) BOOST_AUTO_TEST_CASE(addrman_getaddr) { - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); // Test: Sanity check, GetAddr should never return anything if addrman // is empty. @@ -357,27 +357,25 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); - std::vector<bool> asmap; // use /16 - - BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1, asmap), 40); + BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1, EMPTY_NETGROUPMAN), 40); // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. - BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info1.GetTriedBucket(nKey2, asmap)); + BOOST_CHECK(info1.GetTriedBucket(nKey1, EMPTY_NETGROUPMAN) != info1.GetTriedBucket(nKey2, EMPTY_NETGROUPMAN)); // Test: Two addresses with same IP but different ports can map to // different buckets because they have different keys. AddrInfo info2 = AddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info2.GetTriedBucket(nKey1, asmap)); + BOOST_CHECK(info1.GetTriedBucket(nKey1, EMPTY_NETGROUPMAN) != info2.GetTriedBucket(nKey1, EMPTY_NETGROUPMAN)); std::set<int> buckets; for (int i = 0; i < 255; i++) { AddrInfo infoi = AddrInfo( CAddress(ResolveService("250.1.1." + ToString(i)), NODE_NONE), ResolveIP("250.1.1." + ToString(i))); - int bucket = infoi.GetTriedBucket(nKey1, asmap); + int bucket = infoi.GetTriedBucket(nKey1, EMPTY_NETGROUPMAN); buckets.insert(bucket); } // Test: IP addresses in the same /16 prefix should @@ -389,7 +387,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) AddrInfo infoj = AddrInfo( CAddress(ResolveService("250." + ToString(j) + ".1.1"), NODE_NONE), ResolveIP("250." + ToString(j) + ".1.1")); - int bucket = infoj.GetTriedBucket(nKey1, asmap); + int bucket = infoj.GetTriedBucket(nKey1, EMPTY_NETGROUPMAN); buckets.insert(bucket); } // Test: IP addresses in the different /16 prefix should map to more than @@ -409,27 +407,25 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); - std::vector<bool> asmap; // use /16 - // Test: Make sure the buckets are what we expect - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), 786); - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1, asmap), 786); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, EMPTY_NETGROUPMAN), 786); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1, EMPTY_NETGROUPMAN), 786); // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. - BOOST_CHECK(info1.GetNewBucket(nKey1, asmap) != info1.GetNewBucket(nKey2, asmap)); + BOOST_CHECK(info1.GetNewBucket(nKey1, EMPTY_NETGROUPMAN) != info1.GetNewBucket(nKey2, EMPTY_NETGROUPMAN)); // Test: Ports should not affect bucket placement in the addr AddrInfo info2 = AddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), info2.GetNewBucket(nKey1, asmap)); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, EMPTY_NETGROUPMAN), info2.GetNewBucket(nKey1, EMPTY_NETGROUPMAN)); std::set<int> buckets; for (int i = 0; i < 255; i++) { AddrInfo infoi = AddrInfo( CAddress(ResolveService("250.1.1." + ToString(i)), NODE_NONE), ResolveIP("250.1.1." + ToString(i))); - int bucket = infoi.GetNewBucket(nKey1, asmap); + int bucket = infoi.GetNewBucket(nKey1, EMPTY_NETGROUPMAN); buckets.insert(bucket); } // Test: IP addresses in the same group (\16 prefix for IPv4) should @@ -442,7 +438,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) ResolveService( ToString(250 + (j / 255)) + "." + ToString(j % 256) + ".1.1"), NODE_NONE), ResolveIP("251.4.1.1")); - int bucket = infoj.GetNewBucket(nKey1, asmap); + int bucket = infoj.GetNewBucket(nKey1, EMPTY_NETGROUPMAN); buckets.insert(bucket); } // Test: IP addresses in the same source groups should map to NO MORE @@ -454,7 +450,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) AddrInfo infoj = AddrInfo( CAddress(ResolveService("250.1.1.1"), NODE_NONE), ResolveIP("250." + ToString(p) + ".1.1")); - int bucket = infoj.GetNewBucket(nKey1, asmap); + int bucket = infoj.GetNewBucket(nKey1, EMPTY_NETGROUPMAN); buckets.insert(bucket); } // Test: IP addresses in the different source groups should map to MORE @@ -475,6 +471,9 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) // 101.8.0.0/16 AS8 BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) { + std::vector<bool> asmap = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); + NetGroupManager ngm_asmap{asmap}; + CAddress addr1 = CAddress(ResolveService("250.1.1.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.1.1", 9999), NODE_NONE); @@ -486,27 +485,25 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); - std::vector<bool> asmap = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); - - BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1, asmap), 236); + BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1, ngm_asmap), 236); // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. - BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info1.GetTriedBucket(nKey2, asmap)); + BOOST_CHECK(info1.GetTriedBucket(nKey1, ngm_asmap) != info1.GetTriedBucket(nKey2, ngm_asmap)); // Test: Two addresses with same IP but different ports can map to // different buckets because they have different keys. AddrInfo info2 = AddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info2.GetTriedBucket(nKey1, asmap)); + BOOST_CHECK(info1.GetTriedBucket(nKey1, ngm_asmap) != info2.GetTriedBucket(nKey1, ngm_asmap)); std::set<int> buckets; for (int j = 0; j < 255; j++) { AddrInfo infoj = AddrInfo( CAddress(ResolveService("101." + ToString(j) + ".1.1"), NODE_NONE), ResolveIP("101." + ToString(j) + ".1.1")); - int bucket = infoj.GetTriedBucket(nKey1, asmap); + int bucket = infoj.GetTriedBucket(nKey1, ngm_asmap); buckets.insert(bucket); } // Test: IP addresses in the different /16 prefix MAY map to more than @@ -518,7 +515,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) AddrInfo infoj = AddrInfo( CAddress(ResolveService("250." + ToString(j) + ".1.1"), NODE_NONE), ResolveIP("250." + ToString(j) + ".1.1")); - int bucket = infoj.GetTriedBucket(nKey1, asmap); + int bucket = infoj.GetTriedBucket(nKey1, ngm_asmap); buckets.insert(bucket); } // Test: IP addresses in the different /16 prefix MAY NOT map to more than @@ -528,6 +525,9 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) { + std::vector<bool> asmap = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); + NetGroupManager ngm_asmap{asmap}; + CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE); @@ -538,27 +538,25 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); - std::vector<bool> asmap = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); - // Test: Make sure the buckets are what we expect - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), 795); - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1, asmap), 795); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, ngm_asmap), 795); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1, ngm_asmap), 795); // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. - BOOST_CHECK(info1.GetNewBucket(nKey1, asmap) != info1.GetNewBucket(nKey2, asmap)); + BOOST_CHECK(info1.GetNewBucket(nKey1, ngm_asmap) != info1.GetNewBucket(nKey2, ngm_asmap)); // Test: Ports should not affect bucket placement in the addr AddrInfo info2 = AddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), info2.GetNewBucket(nKey1, asmap)); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, ngm_asmap), info2.GetNewBucket(nKey1, ngm_asmap)); std::set<int> buckets; for (int i = 0; i < 255; i++) { AddrInfo infoi = AddrInfo( CAddress(ResolveService("250.1.1." + ToString(i)), NODE_NONE), ResolveIP("250.1.1." + ToString(i))); - int bucket = infoi.GetNewBucket(nKey1, asmap); + int bucket = infoi.GetNewBucket(nKey1, ngm_asmap); buckets.insert(bucket); } // Test: IP addresses in the same /16 prefix @@ -571,7 +569,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) ResolveService( ToString(250 + (j / 255)) + "." + ToString(j % 256) + ".1.1"), NODE_NONE), ResolveIP("251.4.1.1")); - int bucket = infoj.GetNewBucket(nKey1, asmap); + int bucket = infoj.GetNewBucket(nKey1, ngm_asmap); buckets.insert(bucket); } // Test: IP addresses in the same source /16 prefix should not map to more @@ -583,7 +581,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) AddrInfo infoj = AddrInfo( CAddress(ResolveService("250.1.1.1"), NODE_NONE), ResolveIP("101." + ToString(p) + ".1.1")); - int bucket = infoj.GetNewBucket(nKey1, asmap); + int bucket = infoj.GetNewBucket(nKey1, ngm_asmap); buckets.insert(bucket); } // Test: IP addresses in the different source /16 prefixes usually map to MORE @@ -595,7 +593,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) AddrInfo infoj = AddrInfo( CAddress(ResolveService("250.1.1.1"), NODE_NONE), ResolveIP("250." + ToString(p) + ".1.1")); - int bucket = infoj.GetNewBucket(nKey1, asmap); + int bucket = infoj.GetNewBucket(nKey1, ngm_asmap); buckets.insert(bucket); } // Test: IP addresses in the different source /16 prefixes sometimes map to NO MORE @@ -606,11 +604,12 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) BOOST_AUTO_TEST_CASE(addrman_serialization) { std::vector<bool> asmap1 = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); + NetGroupManager netgroupman{asmap1}; const auto ratio = GetCheckRatio(m_node); - auto addrman_asmap1 = std::make_unique<AddrMan>(asmap1, DETERMINISTIC, ratio); - auto addrman_asmap1_dup = std::make_unique<AddrMan>(asmap1, DETERMINISTIC, ratio); - auto addrman_noasmap = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, ratio); + auto addrman_asmap1 = std::make_unique<AddrMan>(netgroupman, DETERMINISTIC, ratio); + auto addrman_asmap1_dup = std::make_unique<AddrMan>(netgroupman, DETERMINISTIC, ratio); + auto addrman_noasmap = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, ratio); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); @@ -639,8 +638,8 @@ BOOST_AUTO_TEST_CASE(addrman_serialization) BOOST_CHECK(addr_pos1.position != addr_pos3.position); // deserializing non-asmaped peers.dat to asmaped addrman - addrman_asmap1 = std::make_unique<AddrMan>(asmap1, DETERMINISTIC, ratio); - addrman_noasmap = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, ratio); + addrman_asmap1 = std::make_unique<AddrMan>(netgroupman, DETERMINISTIC, ratio); + addrman_noasmap = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, ratio); addrman_noasmap->Add({addr}, default_source); stream << *addrman_noasmap; stream >> *addrman_asmap1; @@ -651,8 +650,8 @@ BOOST_AUTO_TEST_CASE(addrman_serialization) BOOST_CHECK(addr_pos4 == addr_pos2); // used to map to different buckets, now maps to the same bucket. - addrman_asmap1 = std::make_unique<AddrMan>(asmap1, DETERMINISTIC, ratio); - addrman_noasmap = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, ratio); + addrman_asmap1 = std::make_unique<AddrMan>(netgroupman, DETERMINISTIC, ratio); + addrman_noasmap = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, ratio); CAddress addr1 = CAddress(ResolveService("250.1.1.1"), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.2.1.1"), NODE_NONE); addrman_noasmap->Add({addr, addr2}, default_source); @@ -671,7 +670,7 @@ BOOST_AUTO_TEST_CASE(remove_invalid) { // Confirm that invalid addresses are ignored in unserialization. - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); const CAddress new1{ResolveService("5.5.5.5"), NODE_NONE}; @@ -703,14 +702,14 @@ BOOST_AUTO_TEST_CASE(remove_invalid) BOOST_REQUIRE(pos + sizeof(tried2_raw_replacement) <= stream.size()); memcpy(stream.data() + pos, tried2_raw_replacement, sizeof(tried2_raw_replacement)); - addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); stream >> *addrman; BOOST_CHECK_EQUAL(addrman->size(), 2); } BOOST_AUTO_TEST_CASE(addrman_selecttriedcollision) { - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); BOOST_CHECK(addrman->size() == 0); @@ -743,7 +742,7 @@ BOOST_AUTO_TEST_CASE(addrman_selecttriedcollision) BOOST_AUTO_TEST_CASE(addrman_noevict) { - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); // Add 35 addresses. CNetAddr source = ResolveIP("252.2.2.2"); @@ -795,7 +794,7 @@ BOOST_AUTO_TEST_CASE(addrman_noevict) BOOST_AUTO_TEST_CASE(addrman_evictionworks) { - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); BOOST_CHECK(addrman->size() == 0); @@ -865,7 +864,7 @@ static CDataStream AddrmanToStream(const AddrMan& addrman) BOOST_AUTO_TEST_CASE(load_addrman) { - AddrMan addrman{EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)}; + AddrMan addrman{EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)}; CService addr1, addr2, addr3; BOOST_CHECK(Lookup("250.7.1.1", addr1, 8333, false)); @@ -884,7 +883,7 @@ BOOST_AUTO_TEST_CASE(load_addrman) // Test that the de-serialization does not throw an exception. CDataStream ssPeers1 = AddrmanToStream(addrman); bool exceptionThrown = false; - AddrMan addrman1{EMPTY_ASMAP, !DETERMINISTIC, GetCheckRatio(m_node)}; + AddrMan addrman1{EMPTY_NETGROUPMAN, !DETERMINISTIC, GetCheckRatio(m_node)}; BOOST_CHECK(addrman1.size() == 0); try { @@ -901,7 +900,7 @@ BOOST_AUTO_TEST_CASE(load_addrman) // Test that ReadFromStream creates an addrman with the correct number of addrs. CDataStream ssPeers2 = AddrmanToStream(addrman); - AddrMan addrman2{EMPTY_ASMAP, !DETERMINISTIC, GetCheckRatio(m_node)}; + AddrMan addrman2{EMPTY_NETGROUPMAN, !DETERMINISTIC, GetCheckRatio(m_node)}; BOOST_CHECK(addrman2.size() == 0); ReadFromStream(addrman2, ssPeers2); BOOST_CHECK(addrman2.size() == 3); @@ -939,7 +938,7 @@ BOOST_AUTO_TEST_CASE(load_addrman_corrupted) // Test that the de-serialization of corrupted peers.dat throws an exception. CDataStream ssPeers1 = MakeCorruptPeersDat(); bool exceptionThrown = false; - AddrMan addrman1{EMPTY_ASMAP, !DETERMINISTIC, GetCheckRatio(m_node)}; + AddrMan addrman1{EMPTY_NETGROUPMAN, !DETERMINISTIC, GetCheckRatio(m_node)}; BOOST_CHECK(addrman1.size() == 0); try { unsigned char pchMsgTmp[4]; @@ -955,7 +954,7 @@ BOOST_AUTO_TEST_CASE(load_addrman_corrupted) // Test that ReadFromStream fails if peers.dat is corrupt CDataStream ssPeers2 = MakeCorruptPeersDat(); - AddrMan addrman2{EMPTY_ASMAP, !DETERMINISTIC, GetCheckRatio(m_node)}; + AddrMan addrman2{EMPTY_NETGROUPMAN, !DETERMINISTIC, GetCheckRatio(m_node)}; BOOST_CHECK(addrman2.size() == 0); BOOST_CHECK_THROW(ReadFromStream(addrman2, ssPeers2), std::ios_base::failure); } @@ -963,7 +962,7 @@ BOOST_AUTO_TEST_CASE(load_addrman_corrupted) BOOST_AUTO_TEST_CASE(addrman_update_address) { // Tests updating nTime via Connected() and nServices via SetServices() - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); CNetAddr source{ResolveIP("252.2.2.2")}; CAddress addr{CAddress(ResolveService("250.1.1.1", 8333), NODE_NONE)}; diff --git a/src/test/base32_tests.cpp b/src/test/base32_tests.cpp index 5fab7f0d1e..c6109dfeb0 100644 --- a/src/test/base32_tests.cpp +++ b/src/test/base32_tests.cpp @@ -22,20 +22,16 @@ BOOST_AUTO_TEST_CASE(base32_testvectors) BOOST_CHECK_EQUAL(strEnc, vstrOut[i]); strEnc = EncodeBase32(vstrIn[i], false); BOOST_CHECK_EQUAL(strEnc, vstrOutNoPadding[i]); - std::string strDec = DecodeBase32(vstrOut[i]); - BOOST_CHECK_EQUAL(strDec, vstrIn[i]); + auto dec = DecodeBase32(vstrOut[i]); + BOOST_REQUIRE(dec); + BOOST_CHECK_MESSAGE(MakeByteSpan(*dec) == MakeByteSpan(vstrIn[i]), vstrOut[i]); } // Decoding strings with embedded NUL characters should fail - bool failure; - (void)DecodeBase32("invalid\0"s, &failure); // correct size, invalid due to \0 - BOOST_CHECK(failure); - (void)DecodeBase32("AWSX3VPP"s, &failure); // valid - BOOST_CHECK(!failure); - (void)DecodeBase32("AWSX3VPP\0invalid"s, &failure); // correct size, invalid due to \0 - BOOST_CHECK(failure); - (void)DecodeBase32("AWSX3VPPinvalid"s, &failure); // invalid size - BOOST_CHECK(failure); + BOOST_CHECK(!DecodeBase32("invalid\0"s)); // correct size, invalid due to \0 + BOOST_CHECK(DecodeBase32("AWSX3VPP"s)); // valid + BOOST_CHECK(!DecodeBase32("AWSX3VPP\0invalid"s)); // correct size, invalid due to \0 + BOOST_CHECK(!DecodeBase32("AWSX3VPPinvalid"s)); // invalid size } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/base64_tests.cpp b/src/test/base64_tests.cpp index 6ee1b83691..54a02c6bf8 100644 --- a/src/test/base64_tests.cpp +++ b/src/test/base64_tests.cpp @@ -19,8 +19,9 @@ BOOST_AUTO_TEST_CASE(base64_testvectors) { std::string strEnc = EncodeBase64(vstrIn[i]); BOOST_CHECK_EQUAL(strEnc, vstrOut[i]); - std::string strDec = DecodeBase64(strEnc); - BOOST_CHECK_EQUAL(strDec, vstrIn[i]); + auto dec = DecodeBase64(strEnc); + BOOST_REQUIRE(dec); + BOOST_CHECK_MESSAGE(MakeByteSpan(*dec) == MakeByteSpan(vstrIn[i]), vstrOut[i]); } { @@ -34,15 +35,10 @@ BOOST_AUTO_TEST_CASE(base64_testvectors) } // Decoding strings with embedded NUL characters should fail - bool failure; - (void)DecodeBase64("invalid\0"s, &failure); - BOOST_CHECK(failure); - (void)DecodeBase64("nQB/pZw="s, &failure); - BOOST_CHECK(!failure); - (void)DecodeBase64("nQB/pZw=\0invalid"s, &failure); - BOOST_CHECK(failure); - (void)DecodeBase64("nQB/pZw=invalid\0"s, &failure); - BOOST_CHECK(failure); + BOOST_CHECK(!DecodeBase64("invalid\0"s)); + BOOST_CHECK(DecodeBase64("nQB/pZw="s)); + BOOST_CHECK(!DecodeBase64("nQB/pZw=\0invalid"s)); + BOOST_CHECK(!DecodeBase64("nQB/pZw=invalid\0"s)); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index 7c502349b3..82b9617384 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -4,6 +4,7 @@ #include <blockfilter.h> #include <chainparams.h> +#include <consensus/merkle.h> #include <consensus/validation.h> #include <index/blockfilterindex.h> #include <node/miner.h> @@ -18,7 +19,6 @@ using node::BlockAssembler; using node::CBlockTemplate; -using node::IncrementExtraNonce; BOOST_AUTO_TEST_SUITE(blockfilter_index_tests) @@ -76,9 +76,12 @@ CBlock BuildChainTestingSetup::CreateBlock(const CBlockIndex* prev, for (const CMutableTransaction& tx : txns) { block.vtx.push_back(MakeTransactionRef(tx)); } - // IncrementExtraNonce creates a valid coinbase and merkleRoot - unsigned int extraNonce = 0; - IncrementExtraNonce(&block, prev, extraNonce); + { + CMutableTransaction tx_coinbase{*block.vtx.at(0)}; + tx_coinbase.vin.at(0).scriptSig = CScript{} << prev->nHeight + 1; + block.vtx.at(0) = MakeTransactionRef(std::move(tx_coinbase)); + block.hashMerkleRoot = BlockMerkleRoot(block); + } while (!CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce; diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp index 153ccd984b..79d6b94dff 100644 --- a/src/test/checkqueue_tests.cpp +++ b/src/test/checkqueue_tests.cpp @@ -19,13 +19,17 @@ #include <vector> /** - * Identical to TestingSetup but excludes lock contention logging, as some of - * these tests are designed to be heavily contested to trigger race conditions - * or other issues. + * Identical to TestingSetup but excludes lock contention logging if + * `DEBUG_LOCKCONTENTION` is defined, as some of these tests are designed to be + * heavily contested to trigger race conditions or other issues. */ struct NoLockLoggingTestingSetup : public TestingSetup { NoLockLoggingTestingSetup() +#ifdef DEBUG_LOCKCONTENTION : TestingSetup{CBaseChainParams::MAIN, /*extra_args=*/{"-debugexclude=lock"}} {} +#else + : TestingSetup{CBaseChainParams::MAIN} {} +#endif }; BOOST_FIXTURE_TEST_SUITE(checkqueue_tests, NoLockLoggingTestingSetup) @@ -38,7 +42,7 @@ struct FakeCheck { { return true; } - void swap(FakeCheck& x){}; + void swap(FakeCheck& x) noexcept {}; }; struct FakeCheckCheckCompletion { @@ -48,7 +52,7 @@ struct FakeCheckCheckCompletion { n_calls.fetch_add(1, std::memory_order_relaxed); return true; } - void swap(FakeCheckCheckCompletion& x){}; + void swap(FakeCheckCheckCompletion& x) noexcept {}; }; struct FailingCheck { @@ -59,7 +63,7 @@ struct FailingCheck { { return !fails; } - void swap(FailingCheck& x) + void swap(FailingCheck& x) noexcept { std::swap(fails, x.fails); }; @@ -77,7 +81,10 @@ struct UniqueCheck { results.insert(check_id); return true; } - void swap(UniqueCheck& x) { std::swap(x.check_id, check_id); }; + void swap(UniqueCheck& x) noexcept + { + std::swap(x.check_id, check_id); + }; }; @@ -105,7 +112,10 @@ struct MemoryCheck { { fake_allocated_memory.fetch_sub(b, std::memory_order_relaxed); }; - void swap(MemoryCheck& x) { std::swap(b, x.b); }; + void swap(MemoryCheck& x) noexcept + { + std::swap(b, x.b); + }; }; struct FrozenCleanupCheck { @@ -129,7 +139,10 @@ struct FrozenCleanupCheck { cv.wait(l, []{ return nFrozen.load(std::memory_order_relaxed) == 0;}); } } - void swap(FrozenCleanupCheck& x){std::swap(should_freeze, x.should_freeze);}; + void swap(FrozenCleanupCheck& x) noexcept + { + std::swap(should_freeze, x.should_freeze); + }; }; // Static Allocations diff --git a/src/test/coinstatsindex_tests.cpp b/src/test/coinstatsindex_tests.cpp index 92de4ec7ba..5b73481bc1 100644 --- a/src/test/coinstatsindex_tests.cpp +++ b/src/test/coinstatsindex_tests.cpp @@ -2,8 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <chainparams.h> #include <index/coinstatsindex.h> #include <test/util/setup_common.h> +#include <test/util/validation.h> #include <util/time.h> #include <validation.h> @@ -16,6 +18,17 @@ using node::CoinStatsHashType; BOOST_AUTO_TEST_SUITE(coinstatsindex_tests) +static void IndexWaitSynced(BaseIndex& index) +{ + // Allow the CoinStatsIndex to catch up with the block index that is syncing + // in a background thread. + const auto timeout = GetTime<std::chrono::seconds>() + 120s; + while (!index.BlockUntilSyncedToCurrentChain()) { + BOOST_REQUIRE(timeout > GetTime<std::chrono::milliseconds>()); + UninterruptibleSleep(100ms); + } +} + BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup) { CoinStatsIndex coin_stats_index{1 << 20, true}; @@ -36,13 +49,7 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup) BOOST_REQUIRE(coin_stats_index.Start(m_node.chainman->ActiveChainstate())); - // Allow the CoinStatsIndex to catch up with the block index that is syncing - // in a background thread. - const auto timeout = GetTime<std::chrono::seconds>() + 120s; - while (!coin_stats_index.BlockUntilSyncedToCurrentChain()) { - BOOST_REQUIRE(timeout > GetTime<std::chrono::milliseconds>()); - UninterruptibleSleep(100ms); - } + IndexWaitSynced(coin_stats_index); // Check that CoinStatsIndex works for genesis block. const CBlockIndex* genesis_block_index; @@ -78,4 +85,44 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup) // Rest of shutdown sequence and destructors happen in ~TestingSetup() } +// Test shutdown between BlockConnected and ChainStateFlushed notifications, +// make sure index is not corrupted and is able to reload. +BOOST_FIXTURE_TEST_CASE(coinstatsindex_unclean_shutdown, TestChain100Setup) +{ + CChainState& chainstate = Assert(m_node.chainman)->ActiveChainstate(); + const CChainParams& params = Params(); + { + CoinStatsIndex index{1 << 20}; + BOOST_REQUIRE(index.Start(chainstate)); + IndexWaitSynced(index); + std::shared_ptr<const CBlock> new_block; + CBlockIndex* new_block_index = nullptr; + { + const CScript script_pub_key{CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG}; + const CBlock block = this->CreateBlock({}, script_pub_key, chainstate); + + new_block = std::make_shared<CBlock>(block); + + LOCK(cs_main); + BlockValidationState state; + BOOST_CHECK(CheckBlock(block, state, params.GetConsensus())); + BOOST_CHECK(chainstate.AcceptBlock(new_block, state, &new_block_index, true, nullptr, nullptr)); + CCoinsViewCache view(&chainstate.CoinsTip()); + BOOST_CHECK(chainstate.ConnectBlock(block, state, new_block_index, view)); + } + // Send block connected notification, then stop the index without + // sending a chainstate flushed notification. Prior to #24138, this + // would cause the index to be corrupted and fail to reload. + ValidationInterfaceTest::BlockConnected(index, new_block, new_block_index); + index.Stop(); + } + + { + CoinStatsIndex index{1 << 20}; + // Make sure the index can be loaded. + BOOST_REQUIRE(index.Start(chainstate)); + index.Stop(); + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/data/key_io_invalid.json b/src/test/data/key_io_invalid.json index abe07dad24..8f55abfec7 100644 --- a/src/test/data/key_io_invalid.json +++ b/src/test/data/key_io_invalid.json @@ -6,207 +6,207 @@ "x" ], [ - "2v7k5Bb8Lr1MMgTgW6HAf5YHXi6BzpPjHpQ4srD4RSwHYpzXKiXmLAgiLhkXvp3JF5v7nq45EWr" + "1GAdfviErV2Ew95FPtZyikz2qGP3gyCB6Hyu94sedAkPpA523m3fQwps9YKUZkKgQckGPKhRsFR" ], [ - "RAZzCGtMbiUgMiiyrZySrSpdfnQReFXA3r" + "37G2kMDLpmWVhimxRdzwNfE8JFvWXnJYnVcXeeGrek2qumdJuK7XArcVVpRtLLjRra3t64BEPF2" ], [ - "NYamy7tcPQTzoU5iyQojD3sqhiz7zxkvn8" + "giymtio7u7oqWtmC9YnvAEKkLF3JQpAdkEFkVJKYrVDfaLbhaDpX1ihfF2vZmya1i61fwLPC3YQ" ], [ - "geaFG555Ex5nyRf7JjW6Pj2GwZA8KYxtJJLbr1eZhVW75STbYBZeRszy3wg4pkKdF4ez9J4wQiz" + "8iVk9nLM3nYwRuwypjy9NK5rsuZH7BbrQRZ1pgcQmvMnjAgRXD" ], [ - "2Cxmid3c2XQ2zvQ8SA1ha2TKqvqbJS9XFmXRsCneBS3Po7Qqb65z5zNdsoF9AfieXFcpoVPmkmfa" + "cPTVQ1hbo4qdoysf6Jx5GthqucNmdfqt6J2pZRFeXv8Ep7Kmjqud" ], [ - "gaJ7UVge2njVg9tFTetJrtHgruMm7aQDiSAxfHrVEgzK8N2ooagDVmDkdph434xzc4K96Gjyxcs" + "cQbR2Ny85XFBzUMx3Ed6HsTLw2pVruSgPvt5AofnBUnhiv86gYeW" ], [ - "5JN5BEVQPZ3tAiatz1RGXkrJuE3EC6bervMaPb38wTNgEuZCeqp" + "2UB3iG3VJbX2TRrMwm6ssWskgvU9VjFBYSqCzwqkrihCwo7mg4mtS4WuGZgxTKuxf5A3EcotYEymz" ], [ - "3TnFbyUtBRS5rE1KTW81qLVspjJNaB3uu6uuvLjxhZo2DB6PCGh" + "cQe12pqwPR6ExtZKfrKf1q4b3CTh1Qi7MwuvMvzs79nWXDvESfBJ" ], [ - "7UgSZGaMaTc4d2mdEgcGBFiMeS6eMsithGUqvBsKTQdGzD7XQDbMEYo3gojdbXEPbUdFF3CQoK72f" + "tc1qeul5g2xfkvdkrhcfmdursv73ad64jnkjl9c40f" ], [ - "9261wfqQqruNDnBDhbbb4tN9oKA1KpRFHeoYeufyJApVGixyAG4V" + "bt1pq65rzej5glw3ra79gav6fqnx4haa0z257qr3mc8cggkefahmgvyseufhc0" ], [ - "cS824CTUh18scFmYuqt6BgxuRhdR4dEEnCHs3fzBbcyQgbfasHbw" + "tb13hty4qmumlwpp6chxjvcyzza4duqgtmxw3xhm3u9ahj4nyhtwz8eq7ynrj4" ], [ - "tc1q0ywf7wkz6t580n3yemd3ucfw8jxn93tpc6wskt" + "bcrt1r2qxpwuge" ], [ - "bt1pxeeuh96wpm5c6u3kavts2qgwlv6y8um7u7ga6ltlwrhrv7w9vers8lgt3k" + "bc10uexgzna2dpfk0vjt35srz6a27ps6m0l89jweznt83n2sqn2fx4hvn9ym5af8wut34sfrqhk3" ], [ - "tb130lvl2lyugsk2tf3zhwcjjv39dmwt2tt7ytqaexy8edwcuwks6p5scll5kz" + "tb1qum6uh0pt4q253qaf520929737v63w5gf" ], [ - "bcrt1rhsveeudk" + "bcrt1q888ryfgxpvl0k7vum8zpyar2u2sexvdhkf38ue37yknmqq0ycrwpl3w48y" ], [ - "bc10rmfwl8nxdweeyc4sf89t0tn9fv9w6qpyzsnl2r4k48vjqh03qas9asdje0rlr0phru0wqw0p" + "bc1qdsuzmn04k2z8vryw8l4dj8m5ygqgnne5n" ], [ - "tb1qjqnfsuatr54e957xzg9sqk7yqcry9lns" + "tb1qlj8es50nc8j8r8xshrjgzmw5azx89efghmw8ju6zcqla0g6xcnrstsjz7k" ], [ - "bcrt1q8p08mv8echkf3es027u4cdswxlylm3th76ls8v6y4zy4vwsavngpr4e4td" + "bcrt1qzwmyj0z924g7fzs5yvnrkc43y76RVyr2lh5t4r" ], [ - "BC1QNC2H66VLWTWTW52DP0FYUSNU3QQG5VT4V" + "bc1qpu6d26mrulzetu4jqhd7rsunv9aqru26f5c4j8" ], [ - "tb1qgk665m2auw09rc7pqyf7aulcuhmatz9xqtr5mxew7zuysacaascqs9v0vn" + "tb1qun6d26ufh77ghny6u5u8cwz9da7qwc6k4wkuceae9tth06eqlw0syupl4w" ], [ - "bcrt17CAPP7" + "bcrt1qj7g2jps453kj9htk9cxyyc2nxe69x4kzzmth7v" ], [ - "bc1qxmf2d6aerjzam3rur0zufqxqnyqfts5u302s7x" + "bc1p702xksx4z3uqf0u2phllxkfe5cgu0adxptqs0uelx0tqt8e885sqryes2l" ], [ - "tb1qn8x5dnzpexq7nnvrvnhwr9c3wkakpcyu9wwsjzq9pstkwg0t6qhs4l3rv6" + "tb1z7gmh0v6pc30z4xum76lmw8w86yswrlmw" ], [ - "BCRT1Q397G2RNVYRL5LK07CE8NCKHVKP8Z4SC9U0MVH9" + "bcrt1sjsrw6nun4h502cr97xmnyyuhkr22q0s6efrgtu" ], [ - "bc1pgxwyajq0gdn389f69uwn2fw9q0z5c9s063j5dgkdd23ajaud4hpsercr9h" + "2UVPFpGYnLHJezFzjUo42our6PMEoozzRdM" ], [ - "tb1z6mnmp5k542l6yk4ul0mp4rq3yvz44lfm" + "2MygHQjE1U33q3LSC53p69YqFjP8PihumJAF" ], [ - "bcrt17capp7" + "KzNbAQ4mexfAxa6RKBzHQqfoTycaeWpv2p" ], [ - "2D2bqvKseKHdoKjCNvjVULUgmxHu9hjKGwDbPRjTRH59tsHNLeyKwq3vyVBbo9LByY9wiapqjwFY" + "2jDPrDfAKihCGPbPD9ztY8TswAia4V8Bc6vx" ], [ - "2SSjAim4wZpeQRe5zTj1qqS6Li9ttJDaZ3ze" + "4VQUNG1hG64QFtaNyQZQWDdwpxB275Pwb3tvyPt2HDxB8Mi2MgH8Tz3AC83YYiz9LydsLNXEZJLHY" ], [ - "mi9H6MjLwXxy9kxe1x4ToxyLRBsmcZxgVi" + "39TKsUQ5QpEL1wowc6GMUqak94ijirPuP69ooV3xsFmiKQX2dau" ], [ - "VciXoxEitcn88jy197J9n9cpJ1pZahzU3SyWUiHqLgcfjttLEEJz" + "2UEJjT3dSdwc8dAo7oedPzznXceXCEsBbDfAvSymqpqDrkZMv7JBEUpLyhkghioYAWC9W4sKysry" ], [ - "KppmwADGoExPT9Eq5hjRWpWFDbzJyfzHFgsfxBiDHNpVBgWPRNuy" + "7VmMEkphxCFSV1y659Th4dkk6x6bJS5eQvbt8rzUYKQyd6ACgwQ4vXHtXKFUwP2kW3XULipnHJdZ7" ], [ - "TN7EQXMxKffzvHo54yHHu9R4ks9f5gWBW3MMVf5k72zAqrgVK9ys" + "tc1qdlapns4zkn03juf2k9xwwpct209suj6mgcd9gh" ], [ - "92dbrMEYzP5dD5UhQ6maNkCQ4GLG42BM4Gc6XKZzSSMSfosfkkcB" + "bt1psa5eptk29c4jc9yumeseat3a0l5e2fpmw635za2p4gpwdnthueysxga9je" ], [ - "J7VQxPxyzuWEkRstQWpCz2AgysEz1APgnWCEQrFvkN3umAnCrhQF" + "tb13w8c43lykfj3lvm9sgp6dsnfjla3d57cm83seykunf0ltxjc9lt2q4efm4d" ], [ - "tc1qymllj6c96v5qj2504y27ldtner6eh8ldx38t83" + "bcrt1rjqr2tdkm" ], [ - "bt1flep4g" + "bc10lyxwnxa70l270e6fcmxr4x7dtgu2yvy7gzkurwxy4zhdvgaqrrn6pfg2flyhqzy5t5se8yu3" ], [ - "tb13c553hwygcgj48qwmr9f8q0hgdcfklyaye5sxzcpcjnmxv4z506xs90tchn" + "TB1QFDFM763VXVSUNZHQLPWC0Q8FG5LJX6ZN" ], [ - "bcrt1tyddyu" + "bcrt1q60chha7wfwlau4kdr4mlvyeyc8mnnh9dhxk05e0hmrxcuhghefj36uwyha" ], [ - "bc10qssq2mknjqf0glwe2f3587wc4jysvs3f8s6chysae6hcl6fxzdm4wxyyscrl5k9f5qmnf05a" + "bc1gmk9yu" ], [ - "tb1q425lmgvxdgtyl2m6xuu2pc354y4fvgg8" + "tb1ly0q7p" ], [ - "bcrt1q9wp8e5d2u3u4g0pll0cy7smeeuqezdun9xl439n3p2gg4fvgfvk3hu52hj" + "bcrt1qdwttaw38uf42wxw40kwk3u8nguyTQH3hx6jmqp" ], [ - "bc1qrz5acazpue8vl4zsaxn8fxtmeuqmyjkq3" + "bc1qtsvlht6730n04f2mpaj5vv8hrledn5n5ug8c79" ], [ - "tb1qkeuglpgmnex9tv3fr7htzfrh3rwrk23r52rx9halxzmv9fr85lwq0fwhmp" + "tb1dclvmr" ], [ - "bcrt1qd0t2wrhl7s57z99rsyaekpq0dyjcQRSSmz80r4" + "bcrt1q3fqvctqu48wsvggrt09vj0yk2gzzcscdp4h98u" ], [ - "BC1QXLFDUCGX90T3E53PQCNKJ2PK25MSF3VLPMVY6T" + "bc1prklpq7tjcawg89cmwwqr3u5apwav36xa4zz56ady7crsllm6mpnqts7p86" ], [ - "tb1qmycg4zszgnk34vaurx3cu8wpvteg9h40yq6cp52gt26gjel03t3su3x3xu" + "tb1zkm58zyhxz3ffkfgsyprflg543slsl4c4" ], [ - "bcrt1q9hy58r4fnuxqzdqndpmq9pptc9nt2dw3rczf5e" + "bcrt1snzr5kaypnfhpnjanrhd20fhqcjxm3hfh7dw9fu" ], [ - "BC1PA7682NAY6JQSLUWAJYTC0ERWTMW7A4RPWLNTUS32LCXWLHVKKKTQ2UL8CG" + "2GgnYKqBGuA2Mm5GnrPsMTZR81xPhNtgMYoFUZngZGiobhCuUpCaTriUHRcgFreEekNdPAR17q8d" ], [ - "tb1z850dpxnwz2fzae5h2myatj4yvu6rq5xq" + "AZEah8d1EK362okRBS66e8SvdtYkrE8tsX" ], [ - "bcrt1sp525pzjsmpqvcrawjreww36e9keg876skjvpwt" + "gep8xr77FyPW6zYP15RiV9W8nL6w2HyHB16cUDakfyDceMA6ZzUdhJjk2LPuLYHnLkBqkRTTi6z" ], [ - "xcAvW5jurCpzSpLxBKEhCewCgwwuGhqJnC" + "2NDNP7GY59tTJPZTpbkprhM9SR99Nn5rUs7" ], [ - "2Cvv8yp9kXbQt8EKh6Yma95yJ1uwYF9YKXuVhGJyu3dHGVsb2AVpTC62TFACZZ3KDNrALxR2CVNs" + "2Csgzy2T287YAjeU5tFtt1nPshBZAUFQi4WtgaWyZGKSBNnKXHy2Tmxo8QK4Mfdds977ShcDWC5o" ], [ - "niUuL46hCuEVvkAzZKHvD746qbmLmzip9Pv3F6UZV14JxzEXBnTkVxCT4URapChJG6qAEgsZs6G" + "Kwjk3Vy6sdXMQDGWJzaWmqFxUNtWZCX1q4F4Kpt8jNNUoWJUUaTY" ], [ - "2UHHgGfiipzvB8Eumnmvq6SowvrMJimjT3NwwG1839XEiUfwtpSdkUrseNsQuagXv21ce7aZu6yo" + "Svj8kk98bAS9V4L2crmxakbhmnPm3cJ1tJ4Je4yVzDreU8eSTFURS1SPYv5oWEQD8Q9VBDvx5uF" ], [ - "8u9djKu4u6o3bsgeR4BKNnLK3akpo64FYzDAmA9239wKeshgF97" + "KNYsv6v9GtkGeD4WdQnBEJCrPKQm91PTxAbCfXr66LEd4JDmhPWC" ], [ - "TC1QPAARXSLVMXHVRR0474LZXQYZWLGFZYPSFVL9E4" + "2UJ2H2xvAeXmFKfQwMyDoSdQTTPFMNCT3SsoUafBWKzoGP3NsUK1buEgQZG38viyD53jgMdpqfT7" ], [ - "bt1pakek0n2267t9yaksxaczgr2syhv9y3xkx0wnsdwchfa6xkmjtvuqg3kgyr" + "6aLMfayKF4TW4ecn5SEc8FExpyJA2peKxYRGZhes6tQ4NTTzuGy" ], [ - "tb13h83rtwq62udrhwpn87uely7cyxcjrj0azz6a4r3n9s87x5uj98ys6ufp83" + "7VP4FmcebU2thJns9MnXde7LWfuqR5vMizrAuUoq2GcJjzTyA4RHFcPVdZL8PLg1SbpSFdJrvLXoY5" ], [ - "bcrt1rk5vw5qf2" + "tc1q5qdvt99uc92jyz663dtdpfpv6nr67ahmgwcpq2" ], [ - "bc10d3rmtg62h747en5j6fju5g5qyvsransrkty6ghh96pu647wumctejlsngh9pf26cysrys2x2" + "bt1peu3ppd7x796sjjenp09r8cs22rhylqm9lhggk72qp8q22vzft0wq2a0x6j" ], [ - "tb1qajuy2cdwqgmrzc7la85al5cwcq374tsp" + "tb1323z3lnz7dl3kd0nsuh6xy4he9almzl67anxgg3xdzkaxc9rwntlqdhdzd7" ], [ - "bcrt1q3udxvj6x20chqh723mn064mzz65yr56ef00xk8czvu3jnx04ydapzk02s5" + "bcrt1r2gc42sky" ], [ - "bc1qule2szwzyaq4qy0s3aa4mauucyqt6fewe" + "bc10fd889x4hd54tqu2ewg9t4hhft2wl7m6x50av4uswzw46xe6as0xmltfg7vrjfkvm459vld7w" ], [ - "tb1ql0qny5vg9gh5tyzke6dw36px5ulkrp24x53x0pl2t5lpwrtejw3s2seej2" + "TB1QZY7V0F2AT3308YGGNGN66ULJTCN3RY6F" ], [ - "bcrt17CAPP7" + "bcrt1qjg3cwht92znyw0l4r5rtctmls337nrc7g0ry9drjxmlecjd3atl3fake7c" ], [ - "bc1qtvm6davyf725wfedc2d5mrgfewqgcrce8gjrpl" + "bc1qmgf8xt8xkecl79k04mma3lz34gqep7hg4" ], [ - "tb1q5acjgtqrrw3an0dzavxxxzlex8k7aukjzjk9v2u4rmfdqxjphcyq7ge97e" + "TB1Q3F9WGNXE9ZMTTMDN5VKVKHYZ8Y0LCV72YV7V5LSXTJXEYHNHEHASLYL0TZ" ] ] diff --git a/src/test/data/key_io_valid.json b/src/test/data/key_io_valid.json index 5dee44c04b..c051f8b76b 100644 --- a/src/test/data/key_io_valid.json +++ b/src/test/data/key_io_valid.json @@ -1,71 +1,71 @@ [ [ - "1BShJZ8A5q53oJJfMJoEF1gfZCWdZqZwwD", - "76a914728d4cc27d19707b0197cfcd7c412d43287864b588ac", + "1FsSia9rv4NeEwvJ2GvXrX7LyxYspbN2mo", + "76a914a31c06bd463e3923bc1aadbde48b16976c08071788ac", { "chain": "main", "isPrivkey": false } ], [ - "3L1YkZjdeNSqaZcNKZFXQfyokx3zVYm7r6", - "a914c8f37c3cc21561296ad81f4bec6b5de10ebc185187", + "36j4NfKv6Akva9amjWrLG6MuSQym1GuEmm", + "a914373b819a068f32b7a6b38b6b38729647cfde01c287", { "chain": "main", "isPrivkey": false } ], [ - "mhJuoGLgnJC8gdBgBzEigsoyG4omQXejPT", - "76a91413a92d1998e081354d36c13ce0c9dc04b865d40a88ac", + "mzK2FFDEhxqHcmrJw1ysqFkVyhUULo45hZ", + "76a914ce28b26c57472737f5c3561a1761185bd8589a4388ac", { "chain": "test", "isPrivkey": false } ], [ - "2N5VpzKEuYvZJbmg6eUNGnfrrD1ir92FWGu", - "a91486648cc2faaf05660e72c04c7a837bcc3e986f1787", + "2NC2hEhe28ULKAJkW5MjZ3jtTMJdvXmByvK", + "a914ce0bba75891ff9ec60148d4bd4a09ee2dc5c933187", { "chain": "test", "isPrivkey": false } ], [ - "mtQueCtmAnP3E4aBHXCiFNEQAuPaLMuQNy", - "76a9148d74ecd86c845baf9c6d4484d2d00e731b79e34788ac", + "mww4LvqtTMKvmeQvizPz2EQv26xTneWrbg", + "76a914b4110ba93ac54afc14da3bdd19614774a2d55d2988ac", { "chain": "signet", "isPrivkey": false } ], [ - "2NEvWRTHjh89gV52fkperFtwzoFWQiQmiCh", - "a914edc895152c67ccff0ba620bcc373b789ec68266f87", + "2N1r7aC69VHeE7yQJPDLi9T1PYq4wnwvjuT", + "a9145e5a35ab44b3efaea5129ba22b88ba3e2976614587", { "chain": "signet", "isPrivkey": false } ], [ - "mngdx94qJFhSf7A7SAEgQSC9fQJuapujJp", - "76a9144e9dba545455a80ce94c343d1cac9dec62cbf22288ac", + "n4fajahJrAuKbN7uNsKjLjQkz9Qn5ewJXQ", + "76a914fdeca3b08e38af53d7c4c60e3ad208ce5066441088ac", { "chain": "regtest", "isPrivkey": false } ], [ - "2NBzRN3pV56k3JUvSHifaHyzjGHv7ZS9FZZ", - "a914cd9da5642451273e5b6d088854cc1fad4a8d442187", + "2MxFajLApXpYk4VodBSZSt7rw8y4ryABkfA", + "a91436e9f191e0b75036a77f65e2eaa4752443233fbe87", { "chain": "regtest", "isPrivkey": false } ], [ - "5KcrFZvJ2p4dM6QVUPu53cKXcCfozA1PJLHm1mNAxkDYhgThLu4", - "ed6c796e2f62377410766214f55aa81ac9a6590ad7ed57c509c983bf648409ac", + "5JuW2AMDYu4xVwRG9DZW18VbzQrGcd5RCgb99sS6ehJsNQXu5b9", + "8f8943bf956de595665c38ffff23827e17c10cdc1c27a028caae6c9810626198", { "chain": "main", "isCompressed": false, @@ -73,8 +73,8 @@ } ], [ - "L195WBrf2G3nCnun4CLxrb8XKk9LbCqH43THh4n4QrL5SzRzpq9j", - "74f76c106e38d20514a99a86e4fe3bb28319e7dd2ad21dbc170cbb516a5358fa", + "L5nJeqKmpHp4P7F8ZYyjwc5a7P4d8EabuGAzfGJk7yC1BJyzNaEd", + "ff778740f88ddcf102aeb81daee289c044c4a4571c4b6f287400f4b8e0b843f8", { "chain": "main", "isCompressed": true, @@ -82,8 +82,8 @@ } ], [ - "92z6HnMQR4tWqjfVA3UaUN5EuUMgoVMdCa5rZFYZfmgyD7wxYCw", - "b8511e1d74549e305517d48a1d394d1be2cfa5d0f3c0d83f9f450316ffa01276", + "92ZdE5HoLafywnTBbzPxbvRmp75pSfzvdU3XaZGh1cToipgdHVh", + "80c32d81e91bdea04cd7a3819b32275fc3298af4c7ec87eb0099527d041ced5c", { "chain": "test", "isCompressed": false, @@ -91,8 +91,8 @@ } ], [ - "cTPnaF52x4w4Tq6afPxRHux3wbYb86thS7S45A7r3oZc1AHTQ6Qm", - "ad68c48d337181da125de9061933ececcdf7d917631af7d34f7e38082bff9a11", + "cV83kKisF3RQSvXbUCm9ox3kaz5JjEUBWcx8tNydfGJcyeUxuH47", + "e0fcd4ce4e3d0e3de091f21415bb7cd011fac288c42020a879f28c2a4387df9b", { "chain": "test", "isCompressed": true, @@ -100,8 +100,8 @@ } ], [ - "924U35yFcYkxe2JXGmuhSRVaShGyhRDZx1ysPmw1sAHuszGMoxq", - "3e8dfaf78d4f02b11d0b645648a4f3080d71d0d068979c47f7255c9a29eee01d", + "92QuSnywrhsV7WPZChTgSQA23uSmj9MCEEno1eRBDG9sg8M29cX", + "6cf636ed8ac1bab033b64f66feaba65f70e684731e3f39105605968d3a963801", { "chain": "signet", "isCompressed": false, @@ -109,8 +109,8 @@ } ], [ - "cRy1qCf2LUesGPQagTkYwk2V3PyN2KCPKgxeg6k6KoJPzH7nrVjw", - "82d4187690d6b59bcffda27dae52f2ecb87313cfc0904e0b674a27d906a65fde", + "cND53Dhp8eCZqG2ghe8YhSCGesXZ8fE5PGD1khrqNvEi4RBoXhEK", + "12b5a10f3a11e708dc5412833c47ab7c368a21b9efe19293793ec879ce683018", { "chain": "signet", "isCompressed": true, @@ -118,8 +118,8 @@ } ], [ - "932NTcHK35Apf2C3K9Zv1ZdeZEmB1x7ZT2Ju3SjoEY6pUgUpT7H", - "bd7dba24df9e003e145ae9b4862776413a0bb6fa5b4e42753397f2d9536e58a9", + "91mn1wYKEB1zyof1VFm8tMtocZx1oBrKKRCu9GCpgZvPmBLEJjp", + "18a86e5a6c6977ddba0daca7fba5190f67ba56ccdc1b3f31308972236c2e4776", { "chain": "regtest", "isCompressed": false, @@ -127,8 +127,8 @@ } ], [ - "cNa75orYQ2oos52zCnMaS5PG6XbNZKc5LpGxTHacrxwWeX4WAK3E", - "1d87e3c58b08766fea03598380ec8d59f8c88d5392bf683ab1088bd4caf073ee", + "cPisAUdLvqqAr6MYtXnrWvgvyUAwuNyuTvZkDGw6miPhZdaiSDNH", + "3fdfec1371cedcdb8c190ca6ff8ad603f817edc0d93c2a687c7b36dd66e70f2a", { "chain": "regtest", "isCompressed": true, @@ -136,8 +136,8 @@ } ], [ - "bc1q5cuatynjmk4szh40mmunszfzh7zrc5xm9w8ccy", - "0014a639d59272ddab015eafdef9380922bf843c50db", + "bc1qvyq0cc6rahyvsazfdje0twl7ez82ndmuac2lhv", + "00146100fc6343edc8c874496cb2f5bbfec88ea9b77c", { "chain": "main", "isPrivkey": false, @@ -145,8 +145,8 @@ } ], [ - "bc1qkw7lz3ahms6e0ajv27mzh7g62tchjpmve4afc29u7w49tddydy2syv0087", - "0020b3bdf147b7dc3597f64c57b62bf91a52f179076ccd7a9c28bcf3aa55b5a46915", + "bc1qyucykdlhp62tezs0hagqury402qwhk589q80tqs5myh3rxq34nwqhkdhv7", + "002027304b37f70e94bc8a0fbf500e0c957a80ebda87280ef58214d92f119811acdc", { "chain": "main", "isPrivkey": false, @@ -154,8 +154,8 @@ } ], [ - "bc1p5rgvqejqh9dh37t9g94dd9cm8vtqns7dndgj423egwggsggcdzmsspvr7j", - "5120a0d0c06640b95b78f965416ad6971b3b1609c3cd9b512aaa39439088211868b7", + "bc1p83n3au0rjylefxq2nc2xh2y4jzz4pm6zxj4mw5pagdjjr2a9f36s6jjnnu", + "51203c671ef1e3913f94980a9e146ba895908550ef4234abb7503d436521aba54c75", { "chain": "main", "isPrivkey": false, @@ -163,8 +163,8 @@ } ], [ - "bc1zr4pq63udck", - "52021d42", + "bc1z2rksukkjr8", + "520250ed", { "chain": "main", "isPrivkey": false, @@ -172,8 +172,8 @@ } ], [ - "tb1q74fxwnvhsue0l8wremgq66xzvn48jlc5zthsvz", - "0014f552674d978732ff9dc3ced00d68c264ea797f14", + "tb1qcrh3yqn4nlleplcez2yndq2ry8h9ncg3qh7n54", + "0014c0ef1202759fff90ff19128936814321ee59e111", { "chain": "test", "isPrivkey": false, @@ -181,8 +181,8 @@ } ], [ - "tb1qpt7cqgq8ukv92dcraun9c3n0s3aswrt62vtv8nqmkfpa2tjfghesv9ln74", - "00200afd802007e598553703ef265c466f847b070d7a5316c3cc1bb243d52e4945f3", + "tb1quyl9ujpgwr2chdzdnnalen48sup245vdfnh2jxhsuq3yx80rrwlq5hqfe4", + "0020e13e5e482870d58bb44d9cfbfccea78702aad18d4ceea91af0e022431de31bbe", { "chain": "test", "isPrivkey": false, @@ -190,8 +190,8 @@ } ], [ - "tb1ph9v3e8nxct57hknlkhkz75p5pnxnkn05cw8ewpxu6tek56g29xgqydzfu7", - "5120b9591c9e66c2e9ebda7fb5ec2f50340ccd3b4df4c38f9704dcd2f36a690a2990", + "tb1p35n52jy6xkm4wd905tdy8qtagrn73kqdz73xe4zxpvq9t3fp50aqk3s6gz", + "51208d2745489a35b75734afa2da43817d40e7e8d80d17a26cd4460b0055c521a3fa", { "chain": "test", "isPrivkey": false, @@ -199,8 +199,8 @@ } ], [ - "tb1ray6e8gxfx49ers6c4c70l3c8lsxtcmlx", - "5310e93593a0c9354b91c358ae3cffc707fc", + "tb1rgv5m6uvdk3kc7qsuz0c79v88ycr5w4wa", + "53104329bd718db46d8f021c13f1e2b0e726", { "chain": "test", "isPrivkey": false, @@ -208,8 +208,8 @@ } ], [ - "tb1q0sqzfp3zj42u0perxr6jahhu4y03uw4dypk6sc", - "00147c002486229555c7872330f52edefca91f1e3aad", + "tb1q3vya2h5435jkugq2few7dmktlrwq4ejmfaw7kr", + "00148b09d55e958d256e200a4e5de6eecbf8dc0ae65b", { "chain": "signet", "isPrivkey": false, @@ -217,8 +217,8 @@ } ], [ - "tb1q9jv4qnawnuevqaeadn47gkq05ev78m4qg3zqejykdr9u0cm7yutq6gu5dj", - "00202c99504fae9f32c0773d6cebe4580fa659e3eea044440cc89668cbc7e37e2716", + "tb1qxkhrl2s6ttrclckldruea0e8anhrehffl8xv7t0pdyrzm08v2hyqy408nf", + "002035ae3faa1a5ac78fe2df68f99ebf27ecee3cdd29f9cccf2de169062dbcec55c8", { "chain": "signet", "isPrivkey": false, @@ -226,8 +226,8 @@ } ], [ - "tb1pxqf7d825wjtcftj7uep8w24jq3tz8vudfaqj20rns8ahqya56gcs92eqtu", - "51203013e69d54749784ae5ee642772ab2045623b38d4f41253c7381fb7013b4d231", + "tb1pae5um27ahn8n73pgexe3kcwlp8dhswpn684h2k2w6t9a7w3eq65qephd5y", + "5120ee69cdabddbccf3f4428c9b31b61df09db783833d1eb75594ed2cbdf3a3906a8", { "chain": "signet", "isPrivkey": false, @@ -235,8 +235,8 @@ } ], [ - "tb1rsrzkyvu2rt0dcgexajtazlw5nft4j7494ay396q6auw9375wxsrsgag884", - "532080c562338a1adedc2326ec97d17dd49a57597aa5af4912e81aef1c58fa8e3407", + "tb1rx9n9g37az8mu236e5jpxdt0m67y4fuq8rhs0ss3djnm0kscfrwvq0ntlyg", + "532031665447dd11f7c54759a48266adfbd78954f0071de0f8422d94f6fb43091b98", { "chain": "signet", "isPrivkey": false, @@ -244,8 +244,8 @@ } ], [ - "bcrt1qwf52dt9y2sv0f7fwkcpmtfjf74d4np2saeljt6", - "00147268a6aca45418f4f92eb603b5a649f55b598550", + "bcrt1qdavt4j2sd7dlhqsavtnfxvzppw6k7qy97tmnu9", + "00146f58bac9506f9bfb821d62e69330410bb56f0085", { "chain": "regtest", "isPrivkey": false, @@ -253,8 +253,8 @@ } ], [ - "bcrt1q0lma84unycxl4n96etffthqlf7y5axyp4fxf64kmhymvw8l6pwfs39futd", - "00207ff7d3d793260dfaccbacad295dc1f4f894e9881aa4c9d56dbb936c71ffa0b93", + "bcrt1qan8gntac7z7me2ejt4hpru42ad2f759fmy0m3ejvs98656znv7eqga4uhv", + "0020ecce89afb8f0bdbcab325d6e11f2aaeb549f50a9d91fb8e64c814faa685367b2", { "chain": "regtest", "isPrivkey": false, @@ -262,8 +262,8 @@ } ], [ - "bcrt1p3xat2ryucc2v0adrktqnavfzttvezrr27ngltsa2726p2ehvxz4se722v2", - "512089bab50c9cc614c7f5a3b2c13eb1225ad9910c6af4d1f5c3aaf2b41566ec30ab", + "bcrt1pfwxjqvtt4tcxrtdluukfmy2dv7xd2qzdfy6kajv5nwn4yam3wxkq3553uh", + "51204b8d20316baaf061adbfe72c9d914d678cd5004d49356ec9949ba752777171ac", { "chain": "regtest", "isPrivkey": false, @@ -271,8 +271,8 @@ } ], [ - "bcrt1saflydw6e26xhp29euhy5jke5jjqyywk3wvtc9ulgw9dvxyuqy9hdnxthyw755c7ldavy7u", - "6028ea7e46bb59568d70a8b9e5c9495b349480423ad1731782f3e8715ac31380216ed9997723bd4a63df", + "bcrt1sx6p8njlx7h9mc2agz4yg82dzne23050ncq72cneeecez2pst8mahn8xecsf8g6hzx94420", + "6028368279cbe6f5cbbc2ba8154883a9a29e5517d1f3c03cac4f39ce3225060b3efb799cd9c412746ae2", { "chain": "regtest", "isPrivkey": false, @@ -280,72 +280,72 @@ } ], [ - "16y3Q1XVRZqMR9T1XL1FkvNtD2E1bXBuYa", - "76a9144171ec673aeb9fcf42af6094a3c82207e3b9a78188ac", + "1FjL87pn8ky6Vbavd1ZHeChRXtoxwRGCRd", + "76a914a19331b7b2627e663e25a7b001e4c0dcc5e21bc788ac", { "chain": "main", "isPrivkey": false } ], [ - "3CmZZnAiHVQgiAKSakf864oJMxN2BP1eLC", - "a914798575fc1041b9440c4e63c28e57e597d00b7e4387", + "3BZECeAH8gSKkjrTx8PwMrNQBLG18yHpvf", + "a9146c382dcdf5b284760c8e3fead91f7422cd76aa8787", { "chain": "main", "isPrivkey": false } ], [ - "mtCB3SoBo7EYUv8j54kUubGY4x3aJPY8nk", - "76a9148b0c5f9ee714e0d1d24642ad63d9d5f398d9b56588ac", + "n4YNbYuFdPwFrxSP8sjHFbAhUbLMUiY9jE", + "76a914fc8f9851f3c1e4719cd0b8e4816dd4e88c72e52888ac", { "chain": "test", "isPrivkey": false } ], [ - "2N5ymzzKpx6EdUR4UdMZ7t9hcuwqtpHwgw5", - "a9148badb3c3b5c0d39f906f7618e0018b7eae4baf7387", + "2NAeQVZayzVFAtgeC3iYJsjpjWDmsDph71A", + "a914bedc797342c03fd7a346c4c7857ca03d467013b687", { "chain": "test", "isPrivkey": false } ], [ - "myXnpYbub28zgiJupDdZSWZtDbjcyfJVby", - "76a914c59ac57661b57daadd7c0caf7318c14f54c6c0fa88ac", + "mnCBpkNMJEJLehgdEkzSo2eioniyJMxLpZ", + "76a914493c455551e48a1423263b62b127b436106a685488ac", { "chain": "signet", "isPrivkey": false } ], [ - "2MtLg8jS5jSXm9evMzTtvpLjy26dBmjFEoT", - "a9140c0007e89cea625d3bf9543baa5a470bb7e5b67287", + "2N5sNHomeNJDZv67AcFx9ES7FBZY4jx9KDA", + "a9148a776a0f34d56b63e7c595f2b205dbe1c393617a87", { "chain": "signet", "isPrivkey": false } ], [ - "mzCyqdf2UNGdpgkD9NBgLcxdwXRg1i9buY", - "76a914cd04311bdd1ef9c5c24e41930e032aade82a863a88ac", + "mfhE6jAUwjUDNZhaX1PAsDTKfneQF2Nshc", + "76a91401f15a4cc063dae4f4d56b89bfbc8bcc9ae5387c88ac", { "chain": "regtest", "isPrivkey": false } ], [ - "2N3zGiwFku2vQjYnAqXv5Qu2ztfYRhh7tbF", - "a91475d56d75c88e704d6c72fbe84ac1505abf736b4087", + "2MxNm1VHyVU4RuP3u1c1v5aQLk2dQjwy1Qk", + "a91438456f7c076356abadcc67b92ad777eb20fb9f8887", { "chain": "regtest", "isPrivkey": false } ], [ - "5JUHCgyxNSHg64wwju72eNsG6ajqo4Z2fHHw9iLDLfh69rSiL7w", - "5644d06d88855dacf3192a31df8f4acd8e4c155c52a86d2c1fa48303f5cff053", + "5HsL2nZuEebU5nM3RxNVQD9GcAnvNMahqQskf4fkqHe54zwd14e", + "06e8649790a90615a46d22dd762e0c42615336745356c2e16147c0f3d46b40d5", { "chain": "main", "isCompressed": false, @@ -353,8 +353,8 @@ } ], [ - "L2kZaexG69VSriMe9T2m1jkS86iPe3xNbjcdfakRC1PHe7ay78Ji", - "a50ee94aefcabf5a5d7c85be5b3844dee03c5604861dbfc77fe388c91e5a30f8", + "KwuVvu6hsuEMHrfFWJQV64tRrWX3QzqHH18JuAHYqYV6dqBvNKxd", + "147804bf8a0dfff35939a611c7f5a60ac107f33f33d6059f273d2079ab1d90f2", { "chain": "main", "isCompressed": true, @@ -362,8 +362,8 @@ } ], [ - "927JwT1ViCr5TD2ZX8CsMNhg17dXmou5xu4y2KiH54zD7i34UJq", - "4502a54c0026b0150281d41f40860d1e23870c63cdc32645bbed688f2ee41f64", + "921M1RNxghFcsVGqAJksQVbSgx36Yz4u6vebfz1wDujNvgNt93B", + "3777b341c45e2a9b9bf6bfb71dc7d129f64f1b9406ed4f93ade8f56065f1b732", { "chain": "test", "isCompressed": false, @@ -371,8 +371,8 @@ } ], [ - "cTpGGNPVy2Eagawohbr4aGtRJzpLnjxGsGYh9DUcBM45f3KdKGF6", - "ba005a0cb39587aab00bd54c848b59e8adaed47403228567ddc739c2a344ff59", + "cNEnbfF2fcxmmCLWqMAaq6fxJvVkwMbyU3kCbpQznz4Z1j6TZDGb", + "1397b0d4a03e1ab2c54dd9af99ce1ecbfb90c80a58886da95e1181a55703d96b", { "chain": "test", "isCompressed": true, @@ -380,8 +380,8 @@ } ], [ - "932PLCLA19yPNqV67qwHBSGjxi82LVzWBF7josL9ab4Q1kxgPGF", - "bd8677e076eb39770bf7e9f9e8d3f2cf257effab9b4c220fd3439ccfc208c984", + "93BcpCMKPmFCuY8bqS4k3HFrhJ1Afxi4uSsEeJFvX86GYW7PC7W", + "d27d1b6ef55ca2e4d475b5276f2dbb85f7a6459dceeb89c67b776fd3bb974452", { "chain": "signet", "isCompressed": false, @@ -389,8 +389,8 @@ } ], [ - "cViUpEy8URSsLjUvxwL7cEuNgCVqM7oKBzd1ZPbA4khcQsQJuj1j", - "f2b36ade8393e29dc71e52cb75ba1109ba210203cd7d0a5ae881ad6846516203", + "cUtwbyxoL1owPxUafgH2meEpydeywjhnTYv2mJaFHHchz39AaEgy", + "da3ed4ef1647e1733ec076919cab6156077ed9532e7c365acc425747e198b3e1", { "chain": "signet", "isCompressed": true, @@ -398,8 +398,8 @@ } ], [ - "92jddDjJCVDmJtgvBHQ9i58PMash8kwsYhRdNo22ea2MYPXdCBE", - "977bf8686f1bcad28f86c4c14afbd33215746bd19203647bf7ff9c6fddc9cc87", + "927zPWny2SiNaUmHF5NnGQXQWDwbByfFzXGgu88j91ZoutSosvE", + "468e0284f230153db8687d8ec23db079a5b67d72ca04174b3867b13e4ea9945e", { "chain": "regtest", "isCompressed": false, @@ -407,8 +407,8 @@ } ], [ - "cVwAuMoUqRo399X7vXzuzQyPEvZJMXM8c82zHzRkFCxPCSGx8A6y", - "f93acbbce02b8cb9ddca3fad495441e324cc01eb640b0a7b4c9f0e31644c822a", + "cRez45VGSp5EXNqm89K3NJJPSKKapJg5Kbw3atxr2337x2gtgYed", + "798d87586cffbe8c545ab374454e403b1eb831501ebe89f3c3b02f3137bd7b46", { "chain": "regtest", "isCompressed": true, @@ -416,8 +416,8 @@ } ], [ - "bc1qz377zwe5awr68dnggengqx9vrjt05k98q3sw2n", - "0014147de13b34eb87a3b66846668018ac1c96fa58a7", + "bc1qhxt04s5xnpy0kxw4x99n5hpdf5pmtzpqs52es2", + "0014b996fac2869848fb19d5314b3a5c2d4d03b58820", { "chain": "main", "isPrivkey": false, @@ -425,8 +425,8 @@ } ], [ - "bc1qkmhskpdzg8kdkfywhu09kswwn9qan9vnkrf6mk40jvnr06s6sz5ssf82ya", - "0020b6ef0b05a241ecdb248ebf1e5b41ce9941d99593b0d3addaaf932637ea1a80a9", + "bc1qgc9ljrvdf2e0zg9rmmq86xklqwfys7r6wptjlacdgrcdc7sa6ggqu4rrxf", + "0020460bf90d8d4ab2f120a3dec07d1adf039248787a70572ff70d40f0dc7a1dd210", { "chain": "main", "isPrivkey": false, @@ -434,8 +434,8 @@ } ], [ - "bc1ps8cndas60cntk8x79sg9f5e5jz7x050z8agyugln2ukkks23rryqpejzkc", - "512081f136f61a7e26bb1cde2c1054d33490bc67d1e23f504e23f3572d6b415118c8", + "bc1pve739yap4uxjvfk0jrey69078u0gasm2nwvv483ec6zkzulgw9xqu4w9fd", + "5120667d1293a1af0d2626cf90f24d15fe3f1e8ec36a9b98ca9e39c6856173e8714c", { "chain": "main", "isPrivkey": false, @@ -443,8 +443,8 @@ } ], [ - "bc1zn4tsczge9l", - "52029d57", + "bc1zmjtqxkzs89", + "5202dc96", { "chain": "main", "isPrivkey": false, @@ -452,8 +452,8 @@ } ], [ - "tb1q6xw0wwd9n9d7ge87dryz4vm5vtahzhvz6yett3", - "0014d19cf739a5995be464fe68c82ab37462fb715d82", + "tb1ql4k5ayv7p7w0t0ge7tpntgpkgw53g2payxkszr", + "0014fd6d4e919e0f9cf5bd19f2c335a03643a914283d", { "chain": "test", "isPrivkey": false, @@ -461,8 +461,8 @@ } ], [ - "tb1qwn9zq9fu5uk35ykpgsc7rz4uawy4yh0r5m5er26768h5ur50su3qj6evun", - "002074ca20153ca72d1a12c14431e18abceb89525de3a6e991ab5ed1ef4e0e8f8722", + "tb1q9jx3x2qqdpempxrcfgyrkjd5fzeacaqj4ua7cs7fe2sfd2wdaueq5wn26y", + "00202c8d1328006873b098784a083b49b448b3dc7412af3bec43c9caa096a9cdef32", { "chain": "test", "isPrivkey": false, @@ -470,8 +470,8 @@ } ], [ - "tb1pmcdc5d8gr92rtemfsnhpeqanvs0nr82upn5dktxluz9n0qcv34lqxke0wq", - "5120de1b8a34e8195435e76984ee1c83b3641f319d5c0ce8db2cdfe08b37830c8d7e", + "tb1pdswckwd9ym5yf5eyzg8j4jjwnzla8y0tf9cp7aasfkek0u29sz9qfr00yf", + "51206c1d8b39a526e844d324120f2aca4e98bfd391eb49701f77b04db367f145808a", { "chain": "test", "isPrivkey": false, @@ -479,8 +479,8 @@ } ], [ - "tb1rgxjvtfzp0xczz6dlzqv8d5cmuykk4qkk", - "531041a4c5a44179b02169bf101876d31be1", + "tb1r0ecpfxg2udhtc556gqrpwwhk4sw3f0kc", + "53107e7014990ae36ebc529a4006173af6ac", { "chain": "test", "isPrivkey": false, @@ -488,8 +488,8 @@ } ], [ - "tb1qa9dlyt6fydestul4y4wh72yshh044w32np8etk", - "0014e95bf22f49237305f3f5255d7f2890bddf5aba2a", + "tb1q6mwf89hnqhlu8txjgjfs4s7p93ugffn3k062ll", + "0014d6dc9396f305ffc3acd244930ac3c12c7884a671", { "chain": "signet", "isPrivkey": false, @@ -497,8 +497,8 @@ } ], [ - "tb1qu4p26n0033720xm0rjgkds5ehdwf039k2fgv75um5krrvfhrrj7qckl9r2", - "0020e542ad4def8c7ca79b6f1c9166c299bb5c97c4b65250cf539ba5863626e31cbc", + "tb1qafrjalu4d73dql0czau9j6z422434kef235mzljf48ckd5xz3sys09jm97", + "0020ea472eff956fa2d07df8177859685552ab1adb295469b17e49a9f166d0c28c09", { "chain": "signet", "isPrivkey": false, @@ -506,8 +506,8 @@ } ], [ - "tb1pjyukm4n4flwd0ey3lrl06c9kalr60ggmlkcxq2rhhxmy4lvkmkpqexdzqy", - "512091396dd6754fdcd7e491f8fefd60b6efc7a7a11bfdb0602877b9b64afd96dd82", + "tb1pwst9qszjrhuv2e7as0flcq9gm698v6gdxzz9e87p07s8rssdx3zqklm3vf", + "512074165040521df8c567dd83d3fc00a8de8a76690d30845c9fc17fa071c20d3444", { "chain": "signet", "isPrivkey": false, @@ -515,8 +515,8 @@ } ], [ - "tb1r4k75s5syvewsvxufdc3xfhf4tw4u30alw39xny3dnxrl6hau7systymfdv", - "5320adbd485204665d061b896e2264dd355babc8bfbf744a69922d9987fd5fbcf409", + "tb1r3ss76jtsuxe8c8c8lxsehnpak55ylrgr345pww076l536ahjr6jsydamx3", + "53208c21ed4970e1b27c1f07f9a19bcc3db5284f8d038d681739fed7e91d76f21ea5", { "chain": "signet", "isPrivkey": false, @@ -524,8 +524,8 @@ } ], [ - "bcrt1qnk3tdwwj47ppc4pqmxkjdusegedn9ru5gvccwa", - "00149da2b6b9d2af821c5420d9ad26f219465b328f94", + "bcrt1q65nhlm4hf2ptg3t264al57p7wjxj2c3s6kyt83", + "0014d5277feeb74a82b4456ad57bfa783e748d256230", { "chain": "regtest", "isPrivkey": false, @@ -533,8 +533,8 @@ } ], [ - "bcrt1qz7prfshfkwsxuk72pt6mzr6uumq4qllxe4vmwqt89tat48d362yqlykk6a", - "0020178234c2e9b3a06e5bca0af5b10f5ce6c1507fe6cd59b701672afaba9db1d288", + "bcrt1qawvc90lpytw3z3k9etdx54l0exq5f5sqfzu5e45kjnl6slwayeeqx2dyac", + "0020eb9982bfe122dd1146c5cada6a57efc98144d20048b94cd69694ffa87ddd2672", { "chain": "regtest", "isPrivkey": false, @@ -542,8 +542,8 @@ } ], [ - "bcrt1pumee3wj80xvyr7wjmj7zsk26x5pn095aegy862yhx6f2j9sgc9hq6cj4cm", - "5120e6f398ba47799841f9d2dcbc28595a350337969dca087d28973692a91608c16e", + "bcrt1p39a4s4vdcw9kqa8w2t0rp7aj8kfxyw7mce5sk5d70x6wnnmpvt7skf2kxy", + "5120897b58558dc38b6074ee52de30fbb23d92623bdbc6690b51be79b4e9cf6162fd", { "chain": "regtest", "isPrivkey": false, @@ -551,8 +551,8 @@ } ], [ - "bcrt1szqz8hj64d2hhc6nt65v09jxal66pgff2xpcp9kj648qkk8kjzxelsts4dktd799g47uase", - "602810047bcb556aaf7c6a6bd518f2c8ddfeb414252a307012da5aa9c16b1ed211b3f82e156d96df14a8", + "bcrt1s489d9fhmyel0vzfqsrmew4x7r80asuqesm5hgqacy35daflcyufh3j8cgdtflvt99ph05m", + "6028a9cad2a6fb267ef6092080f79754de19dfd8701986e97403b82468dea7f8271378c8f843569fb165", { "chain": "regtest", "isPrivkey": false, @@ -560,48 +560,48 @@ } ], [ - "12agZTajtRE3STSchwWNWnrm467zzTQ916", - "76a9141156e00f70061e5faba8b71593a8c7554b47090c88ac", + "1G9A9j6W8TLuh6dEeVwWeyibK1Uc5MfVFV", + "76a914a614da54daacdb8861f451a0b7e3c27cdf8a099e88ac", { "chain": "main", "isPrivkey": false } ], [ - "3NXqB6iZiPYbKruNT3d9xNBTmtb73xMvvf", - "a914e49decc9e5d97e0547d3642f3a4795b13ae62bca87", + "33GA3ZXbw5o5HeUrBEaqkWXFYYZmdxGRRP", + "a914113ca1afeb49ff3abf176ffa19c2a2b4df19712a87", { "chain": "main", "isPrivkey": false } ], [ - "mjgt4BoCYxjzWvJFoh68x7cj5GeaKDYhyx", - "76a9142dc11fc7b8072f733f690ffb0591c00f4062295c88ac", + "mwgS2HRbjyfYxFnR1nF9VKLvmdgMfFBmGq", + "76a914b14ce7070b53cb0e4b5b5f6e253e876990aeca2e88ac", { "chain": "test", "isPrivkey": false } ], [ - "2NCT6FdQ5MxorHgnFxLeHyGwTGRdkHcrJDH", - "a914d2a8ec992b0894a0d9391ca5d9c45c388c41be7e87", + "2MwBVrJQ76BdaGD76CTmou8cZzQYLpe4NqU", + "a9142b2c149cde619eae3d7fe995243b76a3417541aa87", { "chain": "test", "isPrivkey": false } ], [ - "mpomiA7wqDnMcxaNLC23eBuXAb4U6H4ZqW", - "76a91465e75e340415ed297c58d6a14d3c17ceeaa17bbd88ac", + "mfnJ8tEkqKNFE5YaHTXFxyHk2mnDK2fvDh", + "76a91402e6cd77e649ad8b281271f158fc964ca3f66cb088ac", { "chain": "signet", "isPrivkey": false } ], [ - "2N1pGAA5uatbU2PKvMA9BnJmHcK6yHfMiZa", - "a9145e008b6cc232164570befc23d216060bf4ea793b87", + "2My83D67ir7K8PPzeT6mE2oth3ZwNTVRS9F", + "a9144074d84d32ff62da7b1b3c61925b934bfeb34b0587", { "chain": "signet", "isPrivkey": false diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index a17cc87730..7b492fd1da 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -4,7 +4,6 @@ // Unit tests for denial-of-service detection/prevention code -#include <arith_uint256.h> #include <banman.h> #include <chainparams.h> #include <net.h> @@ -16,7 +15,6 @@ #include <serialize.h> #include <test/util/net.h> #include <test/util/setup_common.h> -#include <txorphanage.h> #include <util/string.h> #include <util/system.h> #include <util/time.h> @@ -34,10 +32,6 @@ static CService ip(uint32_t i) return CService(CNetAddr(s), Params().GetDefaultPort()); } -static NodeId id = 0; - -void UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_seconds); - BOOST_FIXTURE_TEST_SUITE(denialofservice_tests, TestingSetup) // Test eviction of an outbound peer whose chain never advances @@ -51,7 +45,7 @@ BOOST_FIXTURE_TEST_SUITE(denialofservice_tests, TestingSetup) BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) { const CChainParams& chainparams = Params(); - auto connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman); + auto connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman); // Disable inactivity checks for this test to avoid interference static_cast<ConnmanTestMsg*>(connman.get())->SetPeerConnectTimeout(99999s); auto peerLogic = PeerManager::make(chainparams, *connman, *m_node.addrman, nullptr, @@ -59,6 +53,7 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) // Mock an outbound peer CAddress addr1(ip(0xa0b0c001), NODE_NONE); + NodeId id{0}; CNode dummyNode1{id++, ServiceFlags(NODE_NETWORK | NODE_WITNESS), /*sock=*/nullptr, @@ -114,7 +109,7 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) peerLogic->FinalizeNode(dummyNode1); } -static void AddRandomOutboundPeer(std::vector<CNode*>& vNodes, PeerManager& peerLogic, ConnmanTestMsg& connman, ConnectionType connType) +static void AddRandomOutboundPeer(NodeId& id, std::vector<CNode*>& vNodes, PeerManager& peerLogic, ConnmanTestMsg& connman, ConnectionType connType) { CAddress addr(ip(g_insecure_rand_ctx.randbits(32)), NODE_NONE); vNodes.emplace_back(new CNode{id++, @@ -138,8 +133,9 @@ static void AddRandomOutboundPeer(std::vector<CNode*>& vNodes, PeerManager& peer BOOST_AUTO_TEST_CASE(stale_tip_peer_management) { + NodeId id{0}; const CChainParams& chainparams = Params(); - auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman); + auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman); auto peerLogic = PeerManager::make(chainparams, *connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, false); @@ -157,7 +153,7 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) // Mock some outbound peers for (int i = 0; i < max_outbound_full_relay; ++i) { - AddRandomOutboundPeer(vNodes, *peerLogic, *connman, ConnectionType::OUTBOUND_FULL_RELAY); + AddRandomOutboundPeer(id, vNodes, *peerLogic, *connman, ConnectionType::OUTBOUND_FULL_RELAY); } peerLogic->CheckForStaleTipAndEvictPeers(); @@ -183,7 +179,7 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) // on the next check (since we're mocking the time to be in the future, the // required time connected check should be satisfied). SetMockTime(time_init); - AddRandomOutboundPeer(vNodes, *peerLogic, *connman, ConnectionType::OUTBOUND_FULL_RELAY); + AddRandomOutboundPeer(id, vNodes, *peerLogic, *connman, ConnectionType::OUTBOUND_FULL_RELAY); SetMockTime(time_later); peerLogic->CheckForStaleTipAndEvictPeers(); @@ -197,7 +193,7 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) // Update the last announced block time for the last // peer, and check that the next newest node gets evicted. - UpdateLastBlockAnnounceTime(vNodes.back()->GetId(), GetTime()); + peerLogic->UpdateLastBlockAnnounceTime(vNodes.back()->GetId(), GetTime()); peerLogic->CheckForStaleTipAndEvictPeers(); for (int i = 0; i < max_outbound_full_relay - 1; ++i) { @@ -215,8 +211,9 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) BOOST_AUTO_TEST_CASE(block_relay_only_eviction) { + NodeId id{0}; const CChainParams& chainparams = Params(); - auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman); + auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman); auto peerLogic = PeerManager::make(chainparams, *connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, false); @@ -232,7 +229,7 @@ BOOST_AUTO_TEST_CASE(block_relay_only_eviction) // Add block-relay-only peers up to the limit for (int i = 0; i < max_outbound_block_relay; ++i) { - AddRandomOutboundPeer(vNodes, *peerLogic, *connman, ConnectionType::BLOCK_RELAY); + AddRandomOutboundPeer(id, vNodes, *peerLogic, *connman, ConnectionType::BLOCK_RELAY); } peerLogic->CheckForStaleTipAndEvictPeers(); @@ -241,7 +238,7 @@ BOOST_AUTO_TEST_CASE(block_relay_only_eviction) } // Add an extra block-relay-only peer breaking the limit (mocks logic in ThreadOpenConnections) - AddRandomOutboundPeer(vNodes, *peerLogic, *connman, ConnectionType::BLOCK_RELAY); + AddRandomOutboundPeer(id, vNodes, *peerLogic, *connman, ConnectionType::BLOCK_RELAY); peerLogic->CheckForStaleTipAndEvictPeers(); // The extra peer should only get marked for eviction after MINIMUM_CONNECT_TIME @@ -279,7 +276,7 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) { const CChainParams& chainparams = Params(); auto banman = std::make_unique<BanMan>(m_args.GetDataDirBase() / "banlist", nullptr, DEFAULT_MISBEHAVING_BANTIME); - auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman); + auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman); auto peerLogic = PeerManager::make(chainparams, *connman, *m_node.addrman, banman.get(), *m_node.chainman, *m_node.mempool, false); @@ -297,6 +294,7 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) std::array<CNode*, 3> nodes; banman->ClearBanned(); + NodeId id{0}; nodes[0] = new CNode{id++, NODE_NETWORK, /*sock=*/nullptr, @@ -394,7 +392,7 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) { const CChainParams& chainparams = Params(); auto banman = std::make_unique<BanMan>(m_args.GetDataDirBase() / "banlist", nullptr, DEFAULT_MISBEHAVING_BANTIME); - auto connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman); + auto connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman); auto peerLogic = PeerManager::make(chainparams, *connman, *m_node.addrman, banman.get(), *m_node.chainman, *m_node.mempool, false); @@ -403,6 +401,7 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) SetMockTime(nStartTime); // Overrides future calls to GetTime() CAddress addr(ip(0xa0b0c001), NODE_NONE); + NodeId id{0}; CNode dummyNode{id++, NODE_NETWORK, /*sock=*/nullptr, @@ -427,121 +426,4 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) peerLogic->FinalizeNode(dummyNode); } -class TxOrphanageTest : public TxOrphanage -{ -public: - inline size_t CountOrphans() const EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans) - { - return m_orphans.size(); - } - - CTransactionRef RandomOrphan() EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans) - { - std::map<uint256, OrphanTx>::iterator it; - it = m_orphans.lower_bound(InsecureRand256()); - if (it == m_orphans.end()) - it = m_orphans.begin(); - return it->second.tx; - } -}; - -static void MakeNewKeyWithFastRandomContext(CKey& key) -{ - std::vector<unsigned char> keydata; - keydata = g_insecure_rand_ctx.randbytes(32); - key.Set(keydata.data(), keydata.data() + keydata.size(), /*fCompressedIn=*/true); - assert(key.IsValid()); -} - -BOOST_AUTO_TEST_CASE(DoS_mapOrphans) -{ - // This test had non-deterministic coverage due to - // randomly selected seeds. - // This seed is chosen so that all branches of the function - // ecdsa_signature_parse_der_lax are executed during this test. - // Specifically branches that run only when an ECDSA - // signature's R and S values have leading zeros. - g_insecure_rand_ctx = FastRandomContext(ArithToUint256(arith_uint256(33))); - - TxOrphanageTest orphanage; - CKey key; - MakeNewKeyWithFastRandomContext(key); - FillableSigningProvider keystore; - BOOST_CHECK(keystore.AddKey(key)); - - LOCK(g_cs_orphans); - - // 50 orphan transactions: - for (int i = 0; i < 50; i++) - { - CMutableTransaction tx; - tx.vin.resize(1); - tx.vin[0].prevout.n = 0; - tx.vin[0].prevout.hash = InsecureRand256(); - tx.vin[0].scriptSig << OP_1; - tx.vout.resize(1); - tx.vout[0].nValue = 1*CENT; - tx.vout[0].scriptPubKey = GetScriptForDestination(PKHash(key.GetPubKey())); - - orphanage.AddTx(MakeTransactionRef(tx), i); - } - - // ... and 50 that depend on other orphans: - for (int i = 0; i < 50; i++) - { - CTransactionRef txPrev = orphanage.RandomOrphan(); - - CMutableTransaction tx; - tx.vin.resize(1); - tx.vin[0].prevout.n = 0; - tx.vin[0].prevout.hash = txPrev->GetHash(); - tx.vout.resize(1); - tx.vout[0].nValue = 1*CENT; - tx.vout[0].scriptPubKey = GetScriptForDestination(PKHash(key.GetPubKey())); - BOOST_CHECK(SignSignature(keystore, *txPrev, tx, 0, SIGHASH_ALL)); - - orphanage.AddTx(MakeTransactionRef(tx), i); - } - - // This really-big orphan should be ignored: - for (int i = 0; i < 10; i++) - { - CTransactionRef txPrev = orphanage.RandomOrphan(); - - CMutableTransaction tx; - tx.vout.resize(1); - tx.vout[0].nValue = 1*CENT; - tx.vout[0].scriptPubKey = GetScriptForDestination(PKHash(key.GetPubKey())); - tx.vin.resize(2777); - for (unsigned int j = 0; j < tx.vin.size(); j++) - { - tx.vin[j].prevout.n = j; - tx.vin[j].prevout.hash = txPrev->GetHash(); - } - BOOST_CHECK(SignSignature(keystore, *txPrev, tx, 0, SIGHASH_ALL)); - // Re-use same signature for other inputs - // (they don't have to be valid for this test) - for (unsigned int j = 1; j < tx.vin.size(); j++) - tx.vin[j].scriptSig = tx.vin[0].scriptSig; - - BOOST_CHECK(!orphanage.AddTx(MakeTransactionRef(tx), i)); - } - - // Test EraseOrphansFor: - for (NodeId i = 0; i < 3; i++) - { - size_t sizeBefore = orphanage.CountOrphans(); - orphanage.EraseForPeer(i); - BOOST_CHECK(orphanage.CountOrphans() < sizeBefore); - } - - // Test LimitOrphanTxSize() function: - orphanage.LimitOrphans(40); - BOOST_CHECK(orphanage.CountOrphans() <= 40); - orphanage.LimitOrphans(10); - BOOST_CHECK(orphanage.CountOrphans() <= 10); - orphanage.LimitOrphans(0); - BOOST_CHECK(orphanage.CountOrphans() == 0); -} - BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 55a9a78159..63c86a896d 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -311,8 +311,8 @@ void DoCheck(const std::string& prv, const std::string& pub, const std::string& spend.vout.resize(1); std::vector<CTxOut> utxos(1); PrecomputedTransactionData txdata; - txdata.Init(spend, std::move(utxos), /* force */ true); - MutableTransactionSignatureCreator creator(&spend, 0, CAmount{0}, &txdata, SIGHASH_DEFAULT); + txdata.Init(spend, std::move(utxos), /*force=*/true); + MutableTransactionSignatureCreator creator{spend, 0, CAmount{0}, &txdata, SIGHASH_DEFAULT}; SignatureData sigdata; BOOST_CHECK_MESSAGE(ProduceSignature(Merge(keys_priv, script_provider), creator, spks[n], sigdata), prv); } @@ -381,17 +381,17 @@ BOOST_AUTO_TEST_CASE(descriptor_test) Check("wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"00149a1c78a507689f6f54b847ad1cef1e614ee23f1e"}}, OutputType::BECH32); Check("sh(wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "sh(wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", "sh(wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "sh(wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", SIGNABLE, {{"a91484ab21b1b2fd065d4504ff693d832434b6108d7b87"}}, OutputType::P2SH_SEGWIT); Check("tr(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "tr(a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "tr(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "tr(a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE | XONLY_KEYS, {{"512077aab6e066f8a7419c5ab714c12c67d25007ed55a43cadcacb4d7a970a093f11"}}, OutputType::BECH32M); - CheckUnparsable("sh(wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY2))", "sh(wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5))", "Pubkey '03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5' is invalid"); // Invalid pubkey - CheckUnparsable("pkh(deadbeef/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh(deadbeef/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "Key origin start '[ character expected but not found, got 'd' instead"); // Missing start bracket in key origin - CheckUnparsable("pkh([deadbeef]/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh([deadbeef]/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "Multiple ']' characters found for a single pubkey"); // Multiple end brackets in key origin + CheckUnparsable("sh(wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY2))", "sh(wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5))", "wpkh(): Pubkey '03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5' is invalid"); // Invalid pubkey + CheckUnparsable("pkh(deadbeef/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh(deadbeef/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "pkh(): Key origin start '[ character expected but not found, got 'd' instead"); // Missing start bracket in key origin + CheckUnparsable("pkh([deadbeef]/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh([deadbeef]/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "pkh(): Multiple ']' characters found for a single pubkey"); // Multiple end brackets in key origin // Basic single-key uncompressed Check("combo(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "combo(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "combo(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "combo(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)",SIGNABLE, {{"4104a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235ac","76a914b5bd079c4d57cc7fc28ecf8213a6b791625b818388ac"}}, std::nullopt); Check("pk(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "pk(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "pk(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "pk(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", SIGNABLE, {{"4104a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235ac"}}, std::nullopt); Check("pkh(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "pkh(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "pkh(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "pkh(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", SIGNABLE, {{"76a914b5bd079c4d57cc7fc28ecf8213a6b791625b818388ac"}}, OutputType::LEGACY); - CheckUnparsable("wpkh(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "wpkh(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "Uncompressed keys are not allowed"); // No uncompressed keys in witness - CheckUnparsable("wsh(pk(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss))", "wsh(pk(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235))", "Uncompressed keys are not allowed"); // No uncompressed keys in witness - CheckUnparsable("sh(wpkh(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss))", "sh(wpkh(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235))", "Uncompressed keys are not allowed"); // No uncompressed keys in witness + CheckUnparsable("wpkh(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "wpkh(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "wpkh(): Uncompressed keys are not allowed"); // No uncompressed keys in witness + CheckUnparsable("wsh(pk(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss))", "wsh(pk(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235))", "pk(): Uncompressed keys are not allowed"); // No uncompressed keys in witness + CheckUnparsable("sh(wpkh(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss))", "sh(wpkh(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235))", "wpkh(): Uncompressed keys are not allowed"); // No uncompressed keys in witness // Some unconventional single-key constructions Check("sh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "sh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", "sh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "sh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", SIGNABLE, {{"a9141857af51a5e516552b3086430fd8ce55f7c1a52487"}}, OutputType::LEGACY); @@ -415,9 +415,9 @@ BOOST_AUTO_TEST_CASE(descriptor_test) // Mixed range xpubs and const pubkeys Check("multi(1,xprvA2JDeKCSNNZky6uBCviVfJSKyQ1mDYahRjijr5idH2WwLsEd4Hsb2Tyh8RfQMuPh7f7RtyzTtdrbdqqsunu5Mm3wDvUAKRHSC34sJ7in334/*,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)","multi(1,xpub6FHa3pjLCk84BayeJxFW2SP4XRrFd1JYnxeLeU8EqN3vDfZmbqBqaGJAyiLjTAwm6ZLRQUMv1ZACTj37sR62cfN7fe5JnJ7dh8zL4fiyLHV/*,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)","multi(1,xprvA2JDeKCSNNZky6uBCviVfJSKyQ1mDYahRjijr5idH2WwLsEd4Hsb2Tyh8RfQMuPh7f7RtyzTtdrbdqqsunu5Mm3wDvUAKRHSC34sJ7in334/*,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)","multi(1,xpub6FHa3pjLCk84BayeJxFW2SP4XRrFd1JYnxeLeU8EqN3vDfZmbqBqaGJAyiLjTAwm6ZLRQUMv1ZACTj37sR62cfN7fe5JnJ7dh8zL4fiyLHV/*,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", RANGE | MIXED_PUBKEYS, {{"512102df12b7035bdac8e3bab862a3a83d06ea6b17b6753d52edecba9be46f5d09e0762103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd52ae"},{"5121032869a233c9adff9a994e4966e5b821fd5bac066da6c3112488dc52383b4a98ec2103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd52ae"},{"5121035d30b6c66dc1e036c45369da8287518cf7e0d6ed1e2b905171c605708f14ca032103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd52ae"}}, std::nullopt,{{2},{1},{0},{}}); - CheckUnparsable("combo([012345678]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc)", "combo([012345678]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL)", "Fingerprint is not 4 bytes (9 characters instead of 8 characters)"); // Too long key fingerprint - CheckUnparsable("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483648)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483648)", "Key path value 2147483648 is out of range"); // BIP 32 path element overflow - CheckUnparsable("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/1aa)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/1aa)", "Key path value '1aa' is not a valid uint32"); // Path is not valid uint + CheckUnparsable("combo([012345678]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc)", "combo([012345678]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL)", "combo(): Fingerprint is not 4 bytes (9 characters instead of 8 characters)"); // Too long key fingerprint + CheckUnparsable("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483648)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483648)", "pkh(): Key path value 2147483648 is out of range"); // BIP 32 path element overflow + CheckUnparsable("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/1aa)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/1aa)", "pkh(): Key path value '1aa' is not a valid uint32"); // Path is not valid uint Check("pkh([01234567/10/20]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0)", "pkh([01234567/10/20]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0)", "pkh([01234567/10/20/2147483647']xprv9vHkqa6XAPwKqSKSEJMcAB3yoCZhaSVsGZbSkFY5L3Lfjjk8sjZucbsbvEw5o3QrSA69nPfZDCgFnNnLhQ2ohpZuwummndnPasDw2Qr6dC2/0)", "pkh([01234567/10/20/2147483647']xpub69H7F5dQzmVd3vPuLKtcXJziMEQByuDidnX3YdwgtNsecY5HRGtAAQC5mXTt4dsv9RzyjgDjAQs9VGVV6ydYCHnprc9vvaA5YtqWyL6hyds/0)", HARDENED, {{"76a914ebdc90806a9c4356c1c88e42216611e1cb4c1c1788ac"}}, OutputType::LEGACY, {{10, 20, 0xFFFFFFFFUL, 0}}); // Multisig constructions @@ -430,11 +430,11 @@ BOOST_AUTO_TEST_CASE(descriptor_test) Check("sh(wsh(multi(16,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGNz6YCCQtYvFzMtrC6D3tKTKdBBboMrLTsjr2NYVBwapCkn7Mr,KxogYhiNfwxuswvXV66eFyKcCpm7dZ7TqHVqujHAVUjJxyivxQ9X,L2BUNduTSyZwZjwNHynQTF14mv2uz2NRq5n5sYWTb4FkkmqgEE9f,L1okJGHGn1kFjdXHKxXjwVVtmCMR2JA5QsbKCSpSb7ReQjezKeoD,KxDCNSST75HFPaW5QKpzHtAyaCQC7p9Vo3FYfi2u4dXD1vgMiboK,L5edQjFtnkcf5UWURn6UuuoFrabgDQUHdheKCziwN42aLwS3KizU,KzF8UWFcEC7BYTq8Go1xVimMkDmyNYVmXV5PV7RuDicvAocoPB8i,L3nHUboKG2w4VSJ5jYZ5CBM97oeK6YuKvfZxrefdShECcjEYKMWZ,KyjHo36dWkYhimKmVVmQTq3gERv3pnqA4xFCpvUgbGDJad7eS8WE,KwsfyHKRUTZPQtysN7M3tZ4GXTnuov5XRgjdF2XCG8faAPmFruRF,KzCUbGhN9LJhdeFfL9zQgTJMjqxdBKEekRGZX24hXdgCNCijkkap,KzgpMBwwsDLwkaC5UrmBgCYaBD2WgZ7PBoGYXR8KT7gCA9UTN5a3,KyBXTPy4T7YG4q9tcAM3LkvfRpD1ybHMvcJ2ehaWXaSqeGUxEdkP,KzJDe9iwJRPtKP2F2AoN6zBgzS7uiuAwhWCfGdNeYJ3PC1HNJ8M8,L1xbHrxynrqLKkoYc4qtoQPx6uy5qYXR5ZDYVYBSRmCV5piU3JG9)))","sh(wsh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232)))", "sh(wsh(multi(16,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGNz6YCCQtYvFzMtrC6D3tKTKdBBboMrLTsjr2NYVBwapCkn7Mr,KxogYhiNfwxuswvXV66eFyKcCpm7dZ7TqHVqujHAVUjJxyivxQ9X,L2BUNduTSyZwZjwNHynQTF14mv2uz2NRq5n5sYWTb4FkkmqgEE9f,L1okJGHGn1kFjdXHKxXjwVVtmCMR2JA5QsbKCSpSb7ReQjezKeoD,KxDCNSST75HFPaW5QKpzHtAyaCQC7p9Vo3FYfi2u4dXD1vgMiboK,L5edQjFtnkcf5UWURn6UuuoFrabgDQUHdheKCziwN42aLwS3KizU,KzF8UWFcEC7BYTq8Go1xVimMkDmyNYVmXV5PV7RuDicvAocoPB8i,L3nHUboKG2w4VSJ5jYZ5CBM97oeK6YuKvfZxrefdShECcjEYKMWZ,KyjHo36dWkYhimKmVVmQTq3gERv3pnqA4xFCpvUgbGDJad7eS8WE,KwsfyHKRUTZPQtysN7M3tZ4GXTnuov5XRgjdF2XCG8faAPmFruRF,KzCUbGhN9LJhdeFfL9zQgTJMjqxdBKEekRGZX24hXdgCNCijkkap,KzgpMBwwsDLwkaC5UrmBgCYaBD2WgZ7PBoGYXR8KT7gCA9UTN5a3,KyBXTPy4T7YG4q9tcAM3LkvfRpD1ybHMvcJ2ehaWXaSqeGUxEdkP,KzJDe9iwJRPtKP2F2AoN6zBgzS7uiuAwhWCfGdNeYJ3PC1HNJ8M8,L1xbHrxynrqLKkoYc4qtoQPx6uy5qYXR5ZDYVYBSRmCV5piU3JG9)))","sh(wsh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232)))", SIGNABLE, {{"a9147fc63e13dc25e8a95a3cee3d9a714ac3afd96f1e87"}}, OutputType::P2SH_SEGWIT); Check("tr(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,pk(KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy))", "tr(a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,pk(669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0))", "tr(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,pk(KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy))", "tr(a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,pk(669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0))", SIGNABLE | XONLY_KEYS, {{"512017cf18db381d836d8923b1bdb246cfcd818da1a9f0e6e7907f187f0b2f937754"}}, OutputType::BECH32M); CheckUnparsable("sh(multi(16,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGNz6YCCQtYvFzMtrC6D3tKTKdBBboMrLTsjr2NYVBwapCkn7Mr,KxogYhiNfwxuswvXV66eFyKcCpm7dZ7TqHVqujHAVUjJxyivxQ9X,L2BUNduTSyZwZjwNHynQTF14mv2uz2NRq5n5sYWTb4FkkmqgEE9f,L1okJGHGn1kFjdXHKxXjwVVtmCMR2JA5QsbKCSpSb7ReQjezKeoD,KxDCNSST75HFPaW5QKpzHtAyaCQC7p9Vo3FYfi2u4dXD1vgMiboK,L5edQjFtnkcf5UWURn6UuuoFrabgDQUHdheKCziwN42aLwS3KizU,KzF8UWFcEC7BYTq8Go1xVimMkDmyNYVmXV5PV7RuDicvAocoPB8i,L3nHUboKG2w4VSJ5jYZ5CBM97oeK6YuKvfZxrefdShECcjEYKMWZ,KyjHo36dWkYhimKmVVmQTq3gERv3pnqA4xFCpvUgbGDJad7eS8WE,KwsfyHKRUTZPQtysN7M3tZ4GXTnuov5XRgjdF2XCG8faAPmFruRF,KzCUbGhN9LJhdeFfL9zQgTJMjqxdBKEekRGZX24hXdgCNCijkkap,KzgpMBwwsDLwkaC5UrmBgCYaBD2WgZ7PBoGYXR8KT7gCA9UTN5a3,KyBXTPy4T7YG4q9tcAM3LkvfRpD1ybHMvcJ2ehaWXaSqeGUxEdkP,KzJDe9iwJRPtKP2F2AoN6zBgzS7uiuAwhWCfGdNeYJ3PC1HNJ8M8,L1xbHrxynrqLKkoYc4qtoQPx6uy5qYXR5ZDYVYBSRmCV5piU3JG9))","sh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232))", "P2SH script is too large, 547 bytes is larger than 520 bytes"); // P2SH does not fit 16 compressed pubkeys in a redeemscript - CheckUnparsable("wsh(multi(2,[aaaaaaaa][aaaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaa][aaaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multiple ']' characters found for a single pubkey"); // Double key origin descriptor - CheckUnparsable("wsh(multi(2,[aaaagaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaagaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Fingerprint 'aaagaaaa' is not hex"); // Non hex fingerprint - CheckUnparsable("wsh(multi(2,[aaaaaaaa],xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaa],xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "No key provided"); // No public key with origin - CheckUnparsable("wsh(multi(2,[aaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Fingerprint is not 4 bytes (7 characters instead of 8 characters)"); // Too short fingerprint - CheckUnparsable("wsh(multi(2,[aaaaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Fingerprint is not 4 bytes (9 characters instead of 8 characters)"); // Too long fingerprint + CheckUnparsable("wsh(multi(2,[aaaaaaaa][aaaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaa][aaaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multi: Multiple ']' characters found for a single pubkey"); // Double key origin descriptor + CheckUnparsable("wsh(multi(2,[aaaagaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaagaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multi: Fingerprint 'aaagaaaa' is not hex"); // Non hex fingerprint + CheckUnparsable("wsh(multi(2,[aaaaaaaa],xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaa],xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multi: No key provided"); // No public key with origin + CheckUnparsable("wsh(multi(2,[aaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multi: Fingerprint is not 4 bytes (7 characters instead of 8 characters)"); // Too short fingerprint + CheckUnparsable("wsh(multi(2,[aaaaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multi: Fingerprint is not 4 bytes (9 characters instead of 8 characters)"); // Too long fingerprint CheckUnparsable("multi(a,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "multi(a,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "Multi threshold 'a' is not valid"); // Invalid threshold CheckUnparsable("multi(0,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "multi(0,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "Multisig threshold cannot be 0, must be at least 1"); // Threshold of 0 CheckUnparsable("multi(3,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "multi(3,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "Multisig threshold cannot be larger than the number of keys; threshold is 3 but only 2 keys specified"); // Threshold larger than number of keys diff --git a/src/test/fs_tests.cpp b/src/test/fs_tests.cpp index 5875f0218f..d2554483d4 100644 --- a/src/test/fs_tests.cpp +++ b/src/test/fs_tests.cpp @@ -46,8 +46,8 @@ BOOST_AUTO_TEST_CASE(fsbridge_fstream) { fs::path tmpfolder = m_args.GetDataDirBase(); // tmpfile1 should be the same as tmpfile2 - fs::path tmpfile1 = tmpfolder / "fs_tests_₿_🏃"; - fs::path tmpfile2 = tmpfolder / "fs_tests_₿_🏃"; + fs::path tmpfile1 = tmpfolder / fs::u8path("fs_tests_₿_🏃"); + fs::path tmpfile2 = tmpfolder / fs::u8path("fs_tests_₿_🏃"); { std::ofstream file{tmpfile1}; file << "bitcoin"; @@ -86,7 +86,7 @@ BOOST_AUTO_TEST_CASE(fsbridge_fstream) } { // Join an absolute path and a relative path. - fs::path p = fsbridge::AbsPathJoin(tmpfolder, "fs_tests_₿_🏃"); + fs::path p = fsbridge::AbsPathJoin(tmpfolder, fs::u8path("fs_tests_₿_🏃")); BOOST_CHECK(p.is_absolute()); BOOST_CHECK_EQUAL(tmpfile1, p); } diff --git a/src/test/fuzz/addrman.cpp b/src/test/fuzz/addrman.cpp index ba917dec2a..af7a282781 100644 --- a/src/test/fuzz/addrman.cpp +++ b/src/test/fuzz/addrman.cpp @@ -37,11 +37,19 @@ void initialize_addrman() g_setup = testing_setup.get(); } +[[nodiscard]] inline NetGroupManager ConsumeNetGroupManager(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + std::vector<bool> asmap = ConsumeRandomLengthBitVector(fuzzed_data_provider); + if (!SanityCheckASMap(asmap, 128)) asmap.clear(); + return NetGroupManager(asmap); +} + FUZZ_TARGET_INIT(data_stream_addr_man, initialize_addrman) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; CDataStream data_stream = ConsumeDataStream(fuzzed_data_provider); - AddrMan addr_man{/*asmap=*/std::vector<bool>(), /*deterministic=*/false, GetCheckRatio()}; + NetGroupManager netgroupman{ConsumeNetGroupManager(fuzzed_data_provider)}; + AddrMan addr_man(netgroupman, /*deterministic=*/false, GetCheckRatio()); try { ReadFromStream(addr_man, data_stream); } catch (const std::exception&) { @@ -124,8 +132,8 @@ void FillAddrman(AddrMan& addrman, FuzzedDataProvider& fuzzed_data_provider) class AddrManDeterministic : public AddrMan { public: - explicit AddrManDeterministic(std::vector<bool> asmap, FuzzedDataProvider& fuzzed_data_provider) - : AddrMan{std::move(asmap), /*deterministic=*/true, GetCheckRatio()} + explicit AddrManDeterministic(const NetGroupManager& netgroupman, FuzzedDataProvider& fuzzed_data_provider) + : AddrMan(netgroupman, /*deterministic=*/true, GetCheckRatio()) { WITH_LOCK(m_impl->cs, m_impl->insecure_rand = FastRandomContext{ConsumeUInt256(fuzzed_data_provider)}); } @@ -223,19 +231,12 @@ public: } }; -[[nodiscard]] inline std::vector<bool> ConsumeAsmap(FuzzedDataProvider& fuzzed_data_provider) noexcept -{ - std::vector<bool> asmap = ConsumeRandomLengthBitVector(fuzzed_data_provider); - if (!SanityCheckASMap(asmap, 128)) asmap.clear(); - return asmap; -} - FUZZ_TARGET_INIT(addrman, initialize_addrman) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); SetMockTime(ConsumeTime(fuzzed_data_provider)); - std::vector<bool> asmap = ConsumeAsmap(fuzzed_data_provider); - auto addr_man_ptr = std::make_unique<AddrManDeterministic>(asmap, fuzzed_data_provider); + NetGroupManager netgroupman{ConsumeNetGroupManager(fuzzed_data_provider)}; + auto addr_man_ptr = std::make_unique<AddrManDeterministic>(netgroupman, fuzzed_data_provider); if (fuzzed_data_provider.ConsumeBool()) { const std::vector<uint8_t> serialized_data{ConsumeRandomLengthByteVector(fuzzed_data_provider)}; CDataStream ds(serialized_data, SER_DISK, INIT_PROTO_VERSION); @@ -244,7 +245,7 @@ FUZZ_TARGET_INIT(addrman, initialize_addrman) try { ds >> *addr_man_ptr; } catch (const std::ios_base::failure&) { - addr_man_ptr = std::make_unique<AddrManDeterministic>(asmap, fuzzed_data_provider); + addr_man_ptr = std::make_unique<AddrManDeterministic>(netgroupman, fuzzed_data_provider); } } AddrManDeterministic& addr_man = *addr_man_ptr; @@ -313,9 +314,9 @@ FUZZ_TARGET_INIT(addrman_serdeser, initialize_addrman) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); SetMockTime(ConsumeTime(fuzzed_data_provider)); - std::vector<bool> asmap = ConsumeAsmap(fuzzed_data_provider); - AddrManDeterministic addr_man1{asmap, fuzzed_data_provider}; - AddrManDeterministic addr_man2{asmap, fuzzed_data_provider}; + NetGroupManager netgroupman{ConsumeNetGroupManager(fuzzed_data_provider)}; + AddrManDeterministic addr_man1{netgroupman, fuzzed_data_provider}; + AddrManDeterministic addr_man2{netgroupman, fuzzed_data_provider}; CDataStream data_stream(SER_NETWORK, PROTOCOL_VERSION); diff --git a/src/test/fuzz/asmap.cpp b/src/test/fuzz/asmap.cpp index 95be963dc8..1720f8e0ab 100644 --- a/src/test/fuzz/asmap.cpp +++ b/src/test/fuzz/asmap.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <netaddress.h> +#include <netgroup.h> #include <test/fuzz/fuzz.h> #include <util/asmap.h> @@ -56,5 +57,6 @@ FUZZ_TARGET(asmap) memcpy(&ipv4, addr_data, addr_size); net_addr.SetIP(CNetAddr{ipv4}); } - (void)net_addr.GetMappedAS(asmap); + NetGroupManager netgroupman{asmap}; + (void)netgroupman.GetMappedAS(net_addr); } diff --git a/src/test/fuzz/base_encode_decode.cpp b/src/test/fuzz/base_encode_decode.cpp index 196410e29c..48356065b0 100644 --- a/src/test/fuzz/base_encode_decode.cpp +++ b/src/test/fuzz/base_encode_decode.cpp @@ -26,7 +26,7 @@ FUZZ_TARGET_INIT(base_encode_decode, initialize_base_encode_decode) std::vector<unsigned char> decoded; if (DecodeBase58(random_encoded_string, decoded, 100)) { const std::string encoded_string = EncodeBase58(decoded); - assert(encoded_string == TrimString(encoded_string)); + assert(encoded_string == TrimStringView(encoded_string)); assert(ToLower(encoded_string) == ToLower(TrimString(random_encoded_string))); } @@ -36,17 +36,16 @@ FUZZ_TARGET_INIT(base_encode_decode, initialize_base_encode_decode) assert(ToLower(encoded_string) == ToLower(TrimString(random_encoded_string))); } - bool pf_invalid; - std::string decoded_string = DecodeBase32(random_encoded_string, &pf_invalid); - if (!pf_invalid) { - const std::string encoded_string = EncodeBase32(decoded_string); - assert(encoded_string == TrimString(encoded_string)); + auto result = DecodeBase32(random_encoded_string); + if (result) { + const std::string encoded_string = EncodeBase32(*result); + assert(encoded_string == TrimStringView(encoded_string)); assert(ToLower(encoded_string) == ToLower(TrimString(random_encoded_string))); } - decoded_string = DecodeBase64(random_encoded_string, &pf_invalid); - if (!pf_invalid) { - const std::string encoded_string = EncodeBase64(decoded_string); + result = DecodeBase64(random_encoded_string); + if (result) { + const std::string encoded_string = EncodeBase64(*result); assert(encoded_string == TrimString(encoded_string)); assert(ToLower(encoded_string) == ToLower(TrimString(random_encoded_string))); } diff --git a/src/test/fuzz/checkqueue.cpp b/src/test/fuzz/checkqueue.cpp index 0b16f0f0d5..7d107995aa 100644 --- a/src/test/fuzz/checkqueue.cpp +++ b/src/test/fuzz/checkqueue.cpp @@ -26,7 +26,7 @@ struct DumbCheck { return result; } - void swap(DumbCheck& x) + void swap(DumbCheck& x) noexcept { } }; diff --git a/src/test/fuzz/connman.cpp b/src/test/fuzz/connman.cpp index a14d28f4ef..4406779015 100644 --- a/src/test/fuzz/connman.cpp +++ b/src/test/fuzz/connman.cpp @@ -19,12 +19,12 @@ #include <vector> namespace { -const BasicTestingSetup* g_setup; +const TestingSetup* g_setup; } // namespace void initialize_connman() { - static const auto testing_setup = MakeNoLogFileContext<>(); + static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); g_setup = testing_setup.get(); } @@ -32,10 +32,11 @@ FUZZ_TARGET_INIT(connman, initialize_connman) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; SetMockTime(ConsumeTime(fuzzed_data_provider)); - AddrMan addrman(/*asmap=*/std::vector<bool>(), - /*deterministic=*/false, - g_setup->m_node.args->GetIntArg("-checkaddrman", 0)); - CConnman connman{fuzzed_data_provider.ConsumeIntegral<uint64_t>(), fuzzed_data_provider.ConsumeIntegral<uint64_t>(), addrman, fuzzed_data_provider.ConsumeBool()}; + CConnman connman{fuzzed_data_provider.ConsumeIntegral<uint64_t>(), + fuzzed_data_provider.ConsumeIntegral<uint64_t>(), + *g_setup->m_node.addrman, + *g_setup->m_node.netgroupman, + fuzzed_data_provider.ConsumeBool()}; CNetAddr random_netaddr; CNode random_node = ConsumeNode(fuzzed_data_provider); CSubNet random_subnet; diff --git a/src/test/fuzz/crypto_diff_fuzz_chacha20.cpp b/src/test/fuzz/crypto_diff_fuzz_chacha20.cpp index fcc96c6418..1b89d55773 100644 --- a/src/test/fuzz/crypto_diff_fuzz_chacha20.cpp +++ b/src/test/fuzz/crypto_diff_fuzz_chacha20.cpp @@ -128,7 +128,7 @@ void ECRYPT_encrypt_bytes(ECRYPT_ctx* x, const u8* m, u8* c, u32 bytes) { u32 x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15; u32 j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15; - u8* ctarget = NULL; + u8* ctarget = nullptr; u8 tmp[64]; uint32_t i; diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index ed6f172a2a..0a7d0c55bd 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -15,6 +15,7 @@ #include <merkleblock.h> #include <net.h> #include <netbase.h> +#include <netgroup.h> #include <node/utxo_snapshot.h> #include <primitives/block.h> #include <protocol.h> @@ -200,7 +201,8 @@ FUZZ_TARGET_DESERIALIZE(blockmerkleroot, { BlockMerkleRoot(block, &mutated); }) FUZZ_TARGET_DESERIALIZE(addrman_deserialize, { - AddrMan am(/*asmap=*/std::vector<bool>(), + NetGroupManager netgroupman{std::vector<bool>()}; + AddrMan am(netgroupman, /*deterministic=*/false, g_setup->m_node.args->GetIntArg("-checkaddrman", 0)); DeserializeFromFuzzingInput(buffer, am); diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index a490bbfa1d..59adec075e 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -10,7 +10,9 @@ #include <test/util/setup_common.h> #include <util/check.h> #include <util/sock.h> +#include <util/time.h> +#include <csignal> #include <cstdint> #include <exception> #include <fstream> @@ -59,6 +61,7 @@ void FuzzFrameworkRegisterTarget(std::string_view name, TypeTestOneInput target, Assert(it_ins.second); } +static std::string_view g_fuzz_target; static TypeTestOneInput* g_test_one_input{nullptr}; void initialize() @@ -92,9 +95,12 @@ void initialize() should_abort = true; } Assert(!should_abort); - std::string_view fuzz_target{Assert(std::getenv("FUZZ"))}; - const auto it = FuzzTargets().find(fuzz_target); - Assert(it != FuzzTargets().end()); + g_fuzz_target = Assert(std::getenv("FUZZ")); + const auto it = FuzzTargets().find(g_fuzz_target); + if (it == FuzzTargets().end()) { + std::cerr << "No fuzzer for " << g_fuzz_target << "." << std::endl; + std::exit(EXIT_FAILURE); + } Assert(!g_test_one_input); g_test_one_input = &std::get<0>(it->second); std::get<1>(it->second)(); @@ -112,6 +118,35 @@ static bool read_stdin(std::vector<uint8_t>& data) } #endif +#if defined(PROVIDE_FUZZ_MAIN_FUNCTION) && !defined(__AFL_LOOP) +static bool read_file(fs::path p, std::vector<uint8_t>& data) +{ + uint8_t buffer[1024]; + FILE* f = fsbridge::fopen(p, "rb"); + if (f == nullptr) return false; + do { + const size_t length = fread(buffer, sizeof(uint8_t), sizeof(buffer), f); + if (ferror(f)) return false; + data.insert(data.end(), buffer, buffer + length); + } while (!feof(f)); + fclose(f); + return true; +} +#endif + +#if defined(PROVIDE_FUZZ_MAIN_FUNCTION) && !defined(__AFL_LOOP) +static fs::path g_input_path; +void signal_handler(int signal) +{ + if (signal == SIGABRT) { + std::cerr << "Error processing input " << g_input_path << std::endl; + } else { + std::cerr << "Unexpected signal " << signal << " received\n"; + } + std::_Exit(EXIT_FAILURE); +} +#endif + // This function is used by libFuzzer extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { @@ -151,10 +186,37 @@ int main(int argc, char** argv) } #else std::vector<uint8_t> buffer; - if (!read_stdin(buffer)) { + if (argc <= 1) { + if (!read_stdin(buffer)) { + return 0; + } + test_one_input(buffer); return 0; } - test_one_input(buffer); + std::signal(SIGABRT, signal_handler); + int64_t start_time = GetTimeSeconds(); + int tested = 0; + for (int i = 1; i < argc; ++i) { + fs::path input_path(*(argv + i)); + if (fs::is_directory(input_path)) { + for (fs::directory_iterator it(input_path); it != fs::directory_iterator(); ++it) { + if (!fs::is_regular_file(it->path())) continue; + g_input_path = it->path(); + Assert(read_file(it->path(), buffer)); + test_one_input(buffer); + ++tested; + buffer.clear(); + } + } else { + g_input_path = input_path; + Assert(read_file(input_path, buffer)); + test_one_input(buffer); + ++tested; + buffer.clear(); + } + } + int64_t end_time = GetTimeSeconds(); + std::cout << g_fuzz_target << ": succeeded against " << tested << " files in " << (end_time - start_time) << "s." << std::endl; #endif return 0; } diff --git a/src/test/fuzz/http_request.cpp b/src/test/fuzz/http_request.cpp index e3b62032bc..0fe18abaa9 100644 --- a/src/test/fuzz/http_request.cpp +++ b/src/test/fuzz/http_request.cpp @@ -19,23 +19,8 @@ #include <string> #include <vector> -// workaround for libevent versions before 2.1.1, -// when internal functions didn't have underscores at the end -#if LIBEVENT_VERSION_NUMBER < 0x02010100 -extern "C" int evhttp_parse_firstline(struct evhttp_request*, struct evbuffer*); -extern "C" int evhttp_parse_headers(struct evhttp_request*, struct evbuffer*); -inline int evhttp_parse_firstline_(struct evhttp_request* r, struct evbuffer* b) -{ - return evhttp_parse_firstline(r, b); -} -inline int evhttp_parse_headers_(struct evhttp_request* r, struct evbuffer* b) -{ - return evhttp_parse_headers(r, b); -} -#else extern "C" int evhttp_parse_firstline_(struct evhttp_request*, struct evbuffer*); extern "C" int evhttp_parse_headers_(struct evhttp_request*, struct evbuffer*); -#endif std::string RequestMethodString(HTTPRequest::RequestMethod m); @@ -54,7 +39,7 @@ FUZZ_TARGET(http_request) // and is a consequence of our hacky but necessary use of the internal function evhttp_parse_firstline_ in // this fuzzing harness. The workaround is not aesthetically pleasing, but it successfully avoids the troublesome // code path. " http:// HTTP/1.1\n" was a crashing input prior to this workaround. - const std::string http_buffer_str = ToLower({http_buffer.begin(), http_buffer.end()}); + const std::string http_buffer_str = ToLower(std::string{http_buffer.begin(), http_buffer.end()}); if (http_buffer_str.find(" http://") != std::string::npos || http_buffer_str.find(" https://") != std::string::npos || evhttp_parse_firstline_(evreq, evbuf) != 1 || evhttp_parse_headers_(evreq, evbuf) != 1) { evbuffer_free(evbuf); diff --git a/src/test/fuzz/miniscript_decode.cpp b/src/test/fuzz/miniscript_decode.cpp new file mode 100644 index 0000000000..4cc0a1be8f --- /dev/null +++ b/src/test/fuzz/miniscript_decode.cpp @@ -0,0 +1,72 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <core_io.h> +#include <hash.h> +#include <key.h> +#include <script/miniscript.h> +#include <script/script.h> +#include <span.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <util/strencodings.h> + +#include <optional> + +using miniscript::operator""_mst; + + +struct Converter { + typedef CPubKey Key; + + bool ToString(const Key& key, std::string& ret) const { + ret = HexStr(key); + return true; + } + const std::vector<unsigned char> ToPKBytes(const Key& key) const { + return {key.begin(), key.end()}; + } + const std::vector<unsigned char> ToPKHBytes(const Key& key) const { + const auto h = Hash160(key); + return {h.begin(), h.end()}; + } + + template<typename I> + bool FromString(I first, I last, Key& key) const { + const auto bytes = ParseHex(std::string(first, last)); + key.Set(bytes.begin(), bytes.end()); + return key.IsValid(); + } + template<typename I> + bool FromPKBytes(I first, I last, CPubKey& key) const { + key.Set(first, last); + return key.IsValid(); + } + template<typename I> + bool FromPKHBytes(I first, I last, CPubKey& key) const { + assert(last - first == 20); + return false; + } +}; + +const Converter CONVERTER; + +FUZZ_TARGET(miniscript_decode) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const std::optional<CScript> script = ConsumeDeserializable<CScript>(fuzzed_data_provider); + if (!script) return; + + const auto ms = miniscript::FromScript(*script, CONVERTER); + if (!ms) return; + + // We can roundtrip it to its string representation. + std::string ms_str; + assert(ms->ToString(CONVERTER, ms_str)); + assert(*miniscript::FromString(ms_str, CONVERTER) == *ms); + // The Script representation must roundtrip since we parsed it this way the first time. + const CScript ms_script = ms->ToScript(CONVERTER); + assert(ms_script == *script); +} diff --git a/src/test/fuzz/net.cpp b/src/test/fuzz/net.cpp index fb11ea36ce..4981287152 100644 --- a/src/test/fuzz/net.cpp +++ b/src/test/fuzz/net.cpp @@ -52,16 +52,6 @@ FUZZ_TARGET_INIT(net, initialize_net) } }, [&] { - const std::optional<CInv> inv_opt = ConsumeDeserializable<CInv>(fuzzed_data_provider); - if (!inv_opt) { - return; - } - node.AddKnownTx(inv_opt->hash); - }, - [&] { - node.PushTxInventory(ConsumeUInt256(fuzzed_data_provider)); - }, - [&] { const std::optional<CService> service_opt = ConsumeDeserializable<CService>(fuzzed_data_provider); if (!service_opt) { return; diff --git a/src/test/fuzz/node_eviction.cpp b/src/test/fuzz/node_eviction.cpp index 2e90085744..6a363f00f7 100644 --- a/src/test/fuzz/node_eviction.cpp +++ b/src/test/fuzz/node_eviction.cpp @@ -26,7 +26,7 @@ FUZZ_TARGET(node_eviction) /*m_last_block_time=*/std::chrono::seconds{fuzzed_data_provider.ConsumeIntegral<int64_t>()}, /*m_last_tx_time=*/std::chrono::seconds{fuzzed_data_provider.ConsumeIntegral<int64_t>()}, /*fRelevantServices=*/fuzzed_data_provider.ConsumeBool(), - /*fRelayTxes=*/fuzzed_data_provider.ConsumeBool(), + /*m_relay_txs=*/fuzzed_data_provider.ConsumeBool(), /*fBloomFilter=*/fuzzed_data_provider.ConsumeBool(), /*nKeyedNetGroup=*/fuzzed_data_provider.ConsumeIntegral<uint64_t>(), /*prefer_evict=*/fuzzed_data_provider.ConsumeBool(), diff --git a/src/test/fuzz/prevector.cpp b/src/test/fuzz/prevector.cpp index a48bab1ee2..e2d65a4796 100644 --- a/src/test/fuzz/prevector.cpp +++ b/src/test/fuzz/prevector.cpp @@ -161,7 +161,7 @@ public: pre_vector.shrink_to_fit(); } - void swap() + void swap() noexcept { real_vector.swap(real_vector_alt); pre_vector.swap(pre_vector_alt); diff --git a/src/test/fuzz/psbt.cpp b/src/test/fuzz/psbt.cpp index 669688a80d..baa64bba0f 100644 --- a/src/test/fuzz/psbt.cpp +++ b/src/test/fuzz/psbt.cpp @@ -32,7 +32,8 @@ FUZZ_TARGET_INIT(psbt, initialize_psbt) FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; PartiallySignedTransaction psbt_mut; std::string error; - if (!DecodeRawPSBT(psbt_mut, fuzzed_data_provider.ConsumeRandomLengthString(), error)) { + auto str = fuzzed_data_provider.ConsumeRandomLengthString(); + if (!DecodeRawPSBT(psbt_mut, MakeByteSpan(str), error)) { return; } const PartiallySignedTransaction psbt = psbt_mut; @@ -79,7 +80,8 @@ FUZZ_TARGET_INIT(psbt, initialize_psbt) } PartiallySignedTransaction psbt_merge; - if (!DecodeRawPSBT(psbt_merge, fuzzed_data_provider.ConsumeRandomLengthString(), error)) { + str = fuzzed_data_provider.ConsumeRandomLengthString(); + if (!DecodeRawPSBT(psbt_merge, MakeByteSpan(str), error)) { psbt_merge = psbt; } psbt_mut = psbt; diff --git a/src/test/fuzz/script_assets_test_minimizer.cpp b/src/test/fuzz/script_assets_test_minimizer.cpp index 00a3bed12f..a18a0de47e 100644 --- a/src/test/fuzz/script_assets_test_minimizer.cpp +++ b/src/test/fuzz/script_assets_test_minimizer.cpp @@ -11,8 +11,8 @@ #include <streams.h> #include <univalue.h> #include <util/strencodings.h> +#include <util/string.h> -#include <boost/algorithm/string.hpp> #include <cstdint> #include <string> #include <vector> @@ -130,8 +130,7 @@ unsigned int ParseScriptFlags(const std::string& str) if (str.empty()) return 0; unsigned int flags = 0; - std::vector<std::string> words; - boost::algorithm::split(words, str, boost::algorithm::is_any_of(",")); + std::vector<std::string> words = SplitString(str, ','); for (const std::string& word : words) { auto it = FLAG_NAMES.find(word); diff --git a/src/test/fuzz/script_format.cpp b/src/test/fuzz/script_format.cpp index 2fa893f812..9186746bcf 100644 --- a/src/test/fuzz/script_format.cpp +++ b/src/test/fuzz/script_format.cpp @@ -3,7 +3,9 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> +#include <consensus/consensus.h> #include <core_io.h> +#include <policy/policy.h> #include <script/script.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> @@ -19,12 +21,13 @@ FUZZ_TARGET_INIT(script_format, initialize_script_format) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const CScript script{ConsumeScript(fuzzed_data_provider)}; + if (script.size() > MAX_STANDARD_TX_WEIGHT / WITNESS_SCALE_FACTOR) { + return; + } (void)FormatScript(script); (void)ScriptToAsmStr(script, /*fAttemptSighashDecode=*/fuzzed_data_provider.ConsumeBool()); UniValue o1(UniValue::VOBJ); - ScriptPubKeyToUniv(script, o1, /*include_hex=*/fuzzed_data_provider.ConsumeBool()); - UniValue o3(UniValue::VOBJ); - ScriptToUniv(script, o3); + ScriptToUniv(script, /*out=*/o1, /*include_hex=*/fuzzed_data_provider.ConsumeBool(), /*include_address=*/fuzzed_data_provider.ConsumeBool()); } diff --git a/src/test/fuzz/script_sign.cpp b/src/test/fuzz/script_sign.cpp index 1446eafe92..3ddb30d870 100644 --- a/src/test/fuzz/script_sign.cpp +++ b/src/test/fuzz/script_sign.cpp @@ -113,7 +113,7 @@ FUZZ_TARGET_INIT(script_sign, initialize_script_sign) } if (n_in < script_tx_to.vin.size()) { (void)SignSignature(provider, ConsumeScript(fuzzed_data_provider), script_tx_to, n_in, ConsumeMoney(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<int>()); - MutableTransactionSignatureCreator signature_creator{&tx_to, n_in, ConsumeMoney(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<int>()}; + MutableTransactionSignatureCreator signature_creator{tx_to, n_in, ConsumeMoney(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<int>()}; std::vector<unsigned char> vch_sig; CKeyID address; if (fuzzed_data_provider.ConsumeBool()) { diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index ca57af25c4..94399faf04 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -42,7 +42,7 @@ bool LegacyParsePrechecks(const std::string& str) return false; if (str.size() >= 1 && (IsSpace(str[0]) || IsSpace(str[str.size() - 1]))) // No padding allowed return false; - if (!ValidAsCString(str)) // No embedded NUL characters allowed + if (!ContainsNoNUL(str)) // No embedded NUL characters allowed return false; return true; } @@ -188,7 +188,7 @@ FUZZ_TARGET(string) (void)TrimString(random_string_1); (void)TrimString(random_string_1, random_string_2); (void)urlDecode(random_string_1); - (void)ValidAsCString(random_string_1); + (void)ContainsNoNUL(random_string_1); (void)_(random_string_1.c_str()); try { throw scriptnum_error{random_string_1}; @@ -225,6 +225,12 @@ FUZZ_TARGET(string) (void)ParseFixedPoint(random_string_1, fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 1024), &amount_out); } { + const auto single_split{SplitString(random_string_1, fuzzed_data_provider.ConsumeIntegral<char>())}; + assert(single_split.size() >= 1); + const auto any_split{SplitString(random_string_1, random_string_2)}; + assert(any_split.size() >= 1); + } + { (void)Untranslated(random_string_1); const bilingual_str bs1{random_string_1, random_string_2}; const bilingual_str bs2{random_string_2, random_string_1}; diff --git a/src/test/fuzz/transaction.cpp b/src/test/fuzz/transaction.cpp index 6dd8a36692..273aa0dc5c 100644 --- a/src/test/fuzz/transaction.cpp +++ b/src/test/fuzz/transaction.cpp @@ -102,6 +102,6 @@ FUZZ_TARGET_INIT(transaction, initialize_transaction) (void)IsWitnessStandard(tx, coins_view_cache); UniValue u(UniValue::VOBJ); - TxToUniv(tx, /*hashBlock=*/uint256::ZERO, u); - TxToUniv(tx, /*hashBlock=*/uint256::ONE, u); + TxToUniv(tx, /*block_hash=*/uint256::ZERO, /*entry=*/u); + TxToUniv(tx, /*block_hash=*/uint256::ONE, /*entry=*/u); } diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index df5b271d06..f686f4fd86 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -234,14 +234,18 @@ FUZZ_TARGET_INIT(tx_pool_standard, initialize_tx_pool) const bool bypass_limits = fuzzed_data_provider.ConsumeBool(); ::fRequireStandard = fuzzed_data_provider.ConsumeBool(); - // Make sure ProcessNewPackage on one transaction works and always fully validates the transaction. + // Make sure ProcessNewPackage on one transaction works. // The result is not guaranteed to be the same as what is returned by ATMP. const auto result_package = WITH_LOCK(::cs_main, return ProcessNewPackage(chainstate, tx_pool, {tx}, true)); - auto it = result_package.m_tx_results.find(tx->GetWitnessHash()); - Assert(it != result_package.m_tx_results.end()); - Assert(it->second.m_result_type == MempoolAcceptResult::ResultType::VALID || - it->second.m_result_type == MempoolAcceptResult::ResultType::INVALID); + // If something went wrong due to a package-specific policy, it might not return a + // validation result for the transaction. + if (result_package.m_state.GetResult() != PackageValidationResult::PCKG_POLICY) { + auto it = result_package.m_tx_results.find(tx->GetWitnessHash()); + Assert(it != result_package.m_tx_results.end()); + Assert(it->second.m_result_type == MempoolAcceptResult::ResultType::VALID || + it->second.m_result_type == MempoolAcceptResult::ResultType::INVALID); + } const auto res = WITH_LOCK(::cs_main, return AcceptToMemoryPool(chainstate, tx, GetTime(), bypass_limits, /*test_accept=*/false)); const bool accepted = res.m_result_type == MempoolAcceptResult::ResultType::VALID; diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index f0c1b0d147..033c6e18d5 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -193,6 +193,19 @@ int FuzzedSock::GetSockOpt(int level, int opt_name, void* opt_val, socklen_t* op return 0; } +int FuzzedSock::SetSockOpt(int, int, const void*, socklen_t) const +{ + constexpr std::array setsockopt_errnos{ + ENOMEM, + ENOBUFS, + }; + if (m_fuzzed_data_provider.ConsumeBool()) { + SetFuzzedErrNo(m_fuzzed_data_provider, setsockopt_errnos); + return -1; + } + return 0; +} + bool FuzzedSock::Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred) const { constexpr std::array wait_errnos{ @@ -225,7 +238,7 @@ void FillNode(FuzzedDataProvider& fuzzed_data_provider, ConnmanTestMsg& connman, const ServiceFlags remote_services = ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS); const NetPermissionFlags permission_flags = ConsumeWeakEnum(fuzzed_data_provider, ALL_NET_PERMISSION_FLAGS); const int32_t version = fuzzed_data_provider.ConsumeIntegralInRange<int32_t>(MIN_PEER_PROTO_VERSION, std::numeric_limits<int32_t>::max()); - const bool filter_txs = fuzzed_data_provider.ConsumeBool(); + const bool relay_txs{fuzzed_data_provider.ConsumeBool()}; const CNetMsgMaker mm{0}; @@ -241,7 +254,7 @@ void FillNode(FuzzedDataProvider& fuzzed_data_provider, ConnmanTestMsg& connman, uint64_t{1}, // dummy nonce std::string{}, // dummy subver int32_t{}, // dummy starting_height - filter_txs), + relay_txs), }; (void)connman.ReceiveMsgFrom(node, msg_version); @@ -255,10 +268,9 @@ void FillNode(FuzzedDataProvider& fuzzed_data_provider, ConnmanTestMsg& connman, assert(node.nVersion == version); assert(node.GetCommonVersion() == std::min(version, PROTOCOL_VERSION)); assert(node.nServices == remote_services); - if (node.m_tx_relay != nullptr) { - LOCK(node.m_tx_relay->cs_filter); - assert(node.m_tx_relay->fRelayTxes == filter_txs); - } + CNodeStateStats statestats; + assert(peerman.GetNodeStateStats(node.GetId(), statestats)); + assert(statestats.m_relay_txs == (relay_txs && !node.IsBlockOnlyConn())); node.m_permissionFlags = permission_flags; if (successfully_connected) { CSerializedNetMsg msg_verack{mm.Make(NetMsgType::VERACK)}; diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 6c91844633..580105e442 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -68,6 +68,8 @@ public: int GetSockOpt(int level, int opt_name, void* opt_val, socklen_t* opt_len) const override; + int SetSockOpt(int level, int opt_name, const void* opt_val, socklen_t opt_len) const override; + bool Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred = nullptr) const override; bool IsConnected(std::string& errmsg) const override; diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index ef9d72dcde..70dd137e22 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -3,6 +3,8 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <test/util/setup_common.h> +#include <univalue.h> +#include <util/settings.h> #include <util/strencodings.h> #include <util/system.h> @@ -11,7 +13,6 @@ #include <utility> #include <vector> -#include <boost/algorithm/string.hpp> #include <boost/test/unit_test.hpp> BOOST_FIXTURE_TEST_SUITE(getarg_tests, BasicTestingSetup) @@ -19,8 +20,9 @@ BOOST_FIXTURE_TEST_SUITE(getarg_tests, BasicTestingSetup) void ResetArgs(ArgsManager& local_args, const std::string& strArg) { std::vector<std::string> vecArg; - if (strArg.size()) - boost::split(vecArg, strArg, IsSpace, boost::token_compress_on); + if (strArg.size()) { + vecArg = SplitString(strArg, ' '); + } // Insert dummy executable name: vecArg.insert(vecArg.begin(), "testbitcoin"); @@ -41,6 +43,116 @@ void SetupArgs(ArgsManager& local_args, const std::vector<std::pair<std::string, } } +// Test behavior of GetArg functions when string, integer, and boolean types +// are specified in the settings.json file. GetArg functions are convenience +// functions. The GetSetting method can always be used instead of GetArg +// methods to retrieve original values, and there's not always an objective +// answer to what GetArg behavior is best in every case. This test makes sure +// there's test coverage for whatever the current behavior is, so it's not +// broken or changed unintentionally. +BOOST_AUTO_TEST_CASE(setting_args) +{ + ArgsManager args; + SetupArgs(args, {{"-foo", ArgsManager::ALLOW_ANY}}); + + auto set_foo = [&](const util::SettingsValue& value) { + args.LockSettings([&](util::Settings& settings) { + settings.rw_settings["foo"] = value; + }); + }; + + set_foo("str"); + BOOST_CHECK_EQUAL(args.GetSetting("foo").write(), "\"str\""); + BOOST_CHECK_EQUAL(args.GetArg("foo", "default"), "str"); + BOOST_CHECK_EQUAL(args.GetIntArg("foo", 100), 0); + BOOST_CHECK_EQUAL(args.GetBoolArg("foo", true), false); + BOOST_CHECK_EQUAL(args.GetBoolArg("foo", false), false); + + set_foo("99"); + BOOST_CHECK_EQUAL(args.GetSetting("foo").write(), "\"99\""); + BOOST_CHECK_EQUAL(args.GetArg("foo", "default"), "99"); + BOOST_CHECK_EQUAL(args.GetIntArg("foo", 100), 99); + BOOST_CHECK_EQUAL(args.GetBoolArg("foo", true), true); + BOOST_CHECK_EQUAL(args.GetBoolArg("foo", false), true); + + set_foo("3.25"); + BOOST_CHECK_EQUAL(args.GetSetting("foo").write(), "\"3.25\""); + BOOST_CHECK_EQUAL(args.GetArg("foo", "default"), "3.25"); + BOOST_CHECK_EQUAL(args.GetIntArg("foo", 100), 3); + BOOST_CHECK_EQUAL(args.GetBoolArg("foo", true), true); + BOOST_CHECK_EQUAL(args.GetBoolArg("foo", false), true); + + set_foo("0"); + BOOST_CHECK_EQUAL(args.GetSetting("foo").write(), "\"0\""); + BOOST_CHECK_EQUAL(args.GetArg("foo", "default"), "0"); + BOOST_CHECK_EQUAL(args.GetIntArg("foo", 100), 0); + BOOST_CHECK_EQUAL(args.GetBoolArg("foo", true), false); + BOOST_CHECK_EQUAL(args.GetBoolArg("foo", false), false); + + set_foo(""); + BOOST_CHECK_EQUAL(args.GetSetting("foo").write(), "\"\""); + BOOST_CHECK_EQUAL(args.GetArg("foo", "default"), ""); + BOOST_CHECK_EQUAL(args.GetIntArg("foo", 100), 0); + BOOST_CHECK_EQUAL(args.GetBoolArg("foo", true), true); + BOOST_CHECK_EQUAL(args.GetBoolArg("foo", false), true); + + set_foo(99); + BOOST_CHECK_EQUAL(args.GetSetting("foo").write(), "99"); + BOOST_CHECK_EQUAL(args.GetArg("foo", "default"), "99"); + BOOST_CHECK_EQUAL(args.GetIntArg("foo", 100), 99); + BOOST_CHECK_THROW(args.GetBoolArg("foo", true), std::runtime_error); + BOOST_CHECK_THROW(args.GetBoolArg("foo", false), std::runtime_error); + + set_foo(3.25); + BOOST_CHECK_EQUAL(args.GetSetting("foo").write(), "3.25"); + BOOST_CHECK_EQUAL(args.GetArg("foo", "default"), "3.25"); + BOOST_CHECK_THROW(args.GetIntArg("foo", 100), std::runtime_error); + BOOST_CHECK_THROW(args.GetBoolArg("foo", true), std::runtime_error); + BOOST_CHECK_THROW(args.GetBoolArg("foo", false), std::runtime_error); + + set_foo(0); + BOOST_CHECK_EQUAL(args.GetSetting("foo").write(), "0"); + BOOST_CHECK_EQUAL(args.GetArg("foo", "default"), "0"); + BOOST_CHECK_EQUAL(args.GetIntArg("foo", 100), 0); + BOOST_CHECK_THROW(args.GetBoolArg("foo", true), std::runtime_error); + BOOST_CHECK_THROW(args.GetBoolArg("foo", false), std::runtime_error); + + set_foo(true); + BOOST_CHECK_EQUAL(args.GetSetting("foo").write(), "true"); + BOOST_CHECK_EQUAL(args.GetArg("foo", "default"), "1"); + BOOST_CHECK_EQUAL(args.GetIntArg("foo", 100), 1); + BOOST_CHECK_EQUAL(args.GetBoolArg("foo", true), true); + BOOST_CHECK_EQUAL(args.GetBoolArg("foo", false), true); + + set_foo(false); + BOOST_CHECK_EQUAL(args.GetSetting("foo").write(), "false"); + BOOST_CHECK_EQUAL(args.GetArg("foo", "default"), "0"); + BOOST_CHECK_EQUAL(args.GetIntArg("foo", 100), 0); + BOOST_CHECK_EQUAL(args.GetBoolArg("foo", true), false); + BOOST_CHECK_EQUAL(args.GetBoolArg("foo", false), false); + + set_foo(UniValue::VOBJ); + BOOST_CHECK_EQUAL(args.GetSetting("foo").write(), "{}"); + BOOST_CHECK_THROW(args.GetArg("foo", "default"), std::runtime_error); + BOOST_CHECK_THROW(args.GetIntArg("foo", 100), std::runtime_error); + BOOST_CHECK_THROW(args.GetBoolArg("foo", true), std::runtime_error); + BOOST_CHECK_THROW(args.GetBoolArg("foo", false), std::runtime_error); + + set_foo(UniValue::VARR); + BOOST_CHECK_EQUAL(args.GetSetting("foo").write(), "[]"); + BOOST_CHECK_THROW(args.GetArg("foo", "default"), std::runtime_error); + BOOST_CHECK_THROW(args.GetIntArg("foo", 100), std::runtime_error); + BOOST_CHECK_THROW(args.GetBoolArg("foo", true), std::runtime_error); + BOOST_CHECK_THROW(args.GetBoolArg("foo", false), std::runtime_error); + + set_foo(UniValue::VNULL); + BOOST_CHECK_EQUAL(args.GetSetting("foo").write(), "null"); + BOOST_CHECK_EQUAL(args.GetArg("foo", "default"), "default"); + BOOST_CHECK_EQUAL(args.GetIntArg("foo", 100), 100); + BOOST_CHECK_EQUAL(args.GetBoolArg("foo", true), true); + BOOST_CHECK_EQUAL(args.GetBoolArg("foo", false), false); +} + BOOST_AUTO_TEST_CASE(boolarg) { ArgsManager local_args; @@ -245,6 +357,24 @@ BOOST_AUTO_TEST_CASE(patharg) ResetArgs(local_args, "-dir=user/.bitcoin/.//"); BOOST_CHECK_EQUAL(local_args.GetPathArg("-dir"), relative_path); + + // Check negated and default argument handling. Specifying an empty argument + // is the same as not specifying the argument. This is convenient for + // scripting so later command line arguments can override earlier command + // line arguments or bitcoin.conf values. Currently the -dir= case cannot be + // distinguished from -dir case with no assignment, but #16545 would add the + // ability to distinguish these in the future (and treat the no-assign case + // like an imperative command or an error). + ResetArgs(local_args, ""); + BOOST_CHECK_EQUAL(local_args.GetPathArg("-dir", "default"), fs::path{"default"}); + ResetArgs(local_args, "-dir=override"); + BOOST_CHECK_EQUAL(local_args.GetPathArg("-dir", "default"), fs::path{"override"}); + ResetArgs(local_args, "-dir="); + BOOST_CHECK_EQUAL(local_args.GetPathArg("-dir", "default"), fs::path{"default"}); + ResetArgs(local_args, "-dir"); + BOOST_CHECK_EQUAL(local_args.GetPathArg("-dir", "default"), fs::path{"default"}); + ResetArgs(local_args, "-nodir"); + BOOST_CHECK_EQUAL(local_args.GetPathArg("-dir", "default"), fs::path{""}); } BOOST_AUTO_TEST_CASE(doubledash) diff --git a/src/test/httpserver_tests.cpp b/src/test/httpserver_tests.cpp new file mode 100644 index 0000000000..ee59ec6967 --- /dev/null +++ b/src/test/httpserver_tests.cpp @@ -0,0 +1,38 @@ +// Copyright (c) 2012-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <httpserver.h> +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(httpserver_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(test_query_parameters) +{ + std::string uri {}; + + // No parameters + uri = "localhost:8080/rest/headers/someresource.json"; + BOOST_CHECK(!GetQueryParameterFromUri(uri.c_str(), "p1").has_value()); + + // Single parameter + uri = "localhost:8080/rest/endpoint/someresource.json?p1=v1"; + BOOST_CHECK_EQUAL(GetQueryParameterFromUri(uri.c_str(), "p1").value(), "v1"); + BOOST_CHECK(!GetQueryParameterFromUri(uri.c_str(), "p2").has_value()); + + // Multiple parameters + uri = "/rest/endpoint/someresource.json?p1=v1&p2=v2"; + BOOST_CHECK_EQUAL(GetQueryParameterFromUri(uri.c_str(), "p1").value(), "v1"); + BOOST_CHECK_EQUAL(GetQueryParameterFromUri(uri.c_str(), "p2").value(), "v2"); + + // If the query string contains duplicate keys, the first value is returned + uri = "/rest/endpoint/someresource.json?p1=v1&p1=v2"; + BOOST_CHECK_EQUAL(GetQueryParameterFromUri(uri.c_str(), "p1").value(), "v1"); + + // Invalid query string syntax is the same as not having parameters + uri = "/rest/endpoint/someresource.json&p1=v1&p2=v2"; + BOOST_CHECK(!GetQueryParameterFromUri(uri.c_str(), "p1").has_value()); +} +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index 61d334ab18..8cb0515a8a 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -204,7 +204,7 @@ BOOST_AUTO_TEST_CASE(key_key_negation) // create a dummy hash for signature comparison unsigned char rnd[8]; std::string str = "Bitcoin key verification\n"; - GetRandBytes(rnd, sizeof(rnd)); + GetRandBytes(rnd); uint256 hash; CHash256().Write(MakeUCharSpan(str)).Write(rnd).Finalize(hash); diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index d6f24210eb..89424a0cd2 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -747,6 +747,15 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) pool.GetTransactionAncestry(ty6->GetHash(), ancestors, descendants); BOOST_CHECK_EQUAL(ancestors, 9ULL); BOOST_CHECK_EQUAL(descendants, 6ULL); +} + +BOOST_AUTO_TEST_CASE(MempoolAncestryTestsDiamond) +{ + size_t ancestors, descendants; + + CTxMemPool pool; + LOCK2(::cs_main, pool.cs); + TestMemPoolEntryHelper entry; /* Ancestors represented more than once ("diamond") */ // @@ -759,7 +768,6 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) tb = make_tx(/*output_values=*/{5 * COIN, 3 * COIN}, /*inputs=*/ {ta}); tc = make_tx(/*output_values=*/{2 * COIN}, /*inputs=*/{tb}, /*input_indices=*/{1}); td = make_tx(/*output_values=*/{6 * COIN}, /*inputs=*/{tb, tc}, /*input_indices=*/{0, 0}); - pool.clear(); pool.addUnchecked(entry.Fee(10000LL).FromTx(ta)); pool.addUnchecked(entry.Fee(10000LL).FromTx(tb)); pool.addUnchecked(entry.Fee(10000LL).FromTx(tc)); diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index c453dae701..cafa5710fa 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -30,10 +30,12 @@ using node::CBlockTemplate; namespace miner_tests { struct MinerTestingSetup : public TestingSetup { void TestPackageSelection(const CChainParams& chainparams, const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_node.mempool->cs); - bool TestSequenceLocks(const CTransaction& tx, int flags) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_node.mempool->cs) + void TestBasicMining(const CChainParams& chainparams, const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst, int baseheight) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_node.mempool->cs); + void TestPrioritisedMining(const CChainParams& chainparams, const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_node.mempool->cs); + bool TestSequenceLocks(const CTransaction& tx) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_node.mempool->cs) { CCoinsViewMemPool view_mempool(&m_node.chainman->ActiveChainstate().CoinsTip(), *m_node.mempool); - return CheckSequenceLocks(m_node.chainman->ActiveChain().Tip(), view_mempool, tx, flags); + return CheckSequenceLocksAtTip(m_node.chainman->ActiveChain().Tip(), view_mempool, tx); } BlockAssembler AssemblerForTest(const CChainParams& params); }; @@ -191,60 +193,17 @@ void MinerTestingSetup::TestPackageSelection(const CChainParams& chainparams, co BOOST_CHECK(pblocktemplate->block.vtx[8]->GetHash() == hashLowFeeTx2); } -// NOTE: These tests rely on CreateNewBlock doing its own self-validation! -BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) +void MinerTestingSetup::TestBasicMining(const CChainParams& chainparams, const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst, int baseheight) { - // Note that by default, these tests run with size accounting enabled. - const auto chainParams = CreateChainParams(*m_node.args, CBaseChainParams::MAIN); - const CChainParams& chainparams = *chainParams; - CScript scriptPubKey = CScript() << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f") << OP_CHECKSIG; - std::unique_ptr<CBlockTemplate> pblocktemplate; - CMutableTransaction tx; - CScript script; uint256 hash; + CMutableTransaction tx; TestMemPoolEntryHelper entry; entry.nFee = 11; entry.nHeight = 11; - fCheckpointsEnabled = false; - - // Simple block creation, nothing special yet: - BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); - - // We can't make transactions until we have inputs - // Therefore, load 110 blocks :) - static_assert(std::size(BLOCKINFO) == 110, "Should have 110 blocks to import"); - int baseheight = 0; - std::vector<CTransactionRef> txFirst; - for (const auto& bi : BLOCKINFO) { - CBlock *pblock = &pblocktemplate->block; // pointer for convenience - { - LOCK(cs_main); - pblock->nVersion = VERSIONBITS_TOP_BITS; - pblock->nTime = m_node.chainman->ActiveChain().Tip()->GetMedianTimePast()+1; - CMutableTransaction txCoinbase(*pblock->vtx[0]); - txCoinbase.nVersion = 1; - txCoinbase.vin[0].scriptSig = CScript{} << (m_node.chainman->ActiveChain().Height() + 1) << bi.extranonce; - txCoinbase.vout.resize(1); // Ignore the (optional) segwit commitment added by CreateNewBlock (as the hardcoded nonces don't account for this) - txCoinbase.vout[0].scriptPubKey = CScript(); - pblock->vtx[0] = MakeTransactionRef(std::move(txCoinbase)); - if (txFirst.size() == 0) - baseheight = m_node.chainman->ActiveChain().Height(); - if (txFirst.size() < 4) - txFirst.push_back(pblock->vtx[0]); - pblock->hashMerkleRoot = BlockMerkleRoot(*pblock); - pblock->nNonce = bi.nonce; - } - std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(*pblock); - BOOST_CHECK(Assert(m_node.chainman)->ProcessNewBlock(chainparams, shared_pblock, true, nullptr)); - pblock->hashPrevBlock = pblock->GetHash(); - } - - LOCK(cs_main); - LOCK(m_node.mempool->cs); - // Just to make sure we can still make simple blocks - BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); + auto pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); + BOOST_CHECK(pblocktemplate); const CAmount BLOCKSUBSIDY = 50*COIN; const CAmount LOWFEE = CENT; @@ -386,7 +345,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.vin[0].prevout.n = 0; tx.vin[0].scriptSig = CScript() << OP_1; tx.vout[0].nValue = BLOCKSUBSIDY-LOWFEE; - script = CScript() << OP_0; + CScript script = CScript() << OP_0; tx.vout[0].scriptPubKey = GetScriptForDestination(ScriptHash(script)); hash = tx.GetHash(); m_node.mempool->addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); @@ -410,7 +369,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) // non-final txs in mempool SetMockTime(m_node.chainman->ActiveChain().Tip()->GetMedianTimePast()+1); - int flags = LOCKTIME_VERIFY_SEQUENCE|LOCKTIME_MEDIAN_TIME_PAST; + const int flags{LOCKTIME_VERIFY_SEQUENCE | LOCKTIME_MEDIAN_TIME_PAST}; // height map std::vector<int> prevheights; @@ -429,8 +388,8 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.nLockTime = 0; hash = tx.GetHash(); m_node.mempool->addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); - BOOST_CHECK(CheckFinalTx(m_node.chainman->ActiveChain().Tip(), CTransaction(tx), flags)); // Locktime passes - BOOST_CHECK(!TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks fail + BOOST_CHECK(CheckFinalTxAtTip(m_node.chainman->ActiveChain().Tip(), CTransaction{tx})); // Locktime passes + BOOST_CHECK(!TestSequenceLocks(CTransaction{tx})); // Sequence locks fail { CBlockIndex* active_chain_tip = m_node.chainman->ActiveChain().Tip(); @@ -443,19 +402,21 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) prevheights[0] = baseheight + 2; hash = tx.GetHash(); m_node.mempool->addUnchecked(entry.Time(GetTime()).FromTx(tx)); - BOOST_CHECK(CheckFinalTx(m_node.chainman->ActiveChain().Tip(), CTransaction(tx), flags)); // Locktime passes - BOOST_CHECK(!TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks fail - - for (int i = 0; i < CBlockIndex::nMedianTimeSpan; i++) - m_node.chainman->ActiveChain().Tip()->GetAncestor(m_node.chainman->ActiveChain().Tip()->nHeight - i)->nTime += 512; //Trick the MedianTimePast + BOOST_CHECK(CheckFinalTxAtTip(m_node.chainman->ActiveChain().Tip(), CTransaction{tx})); // Locktime passes + BOOST_CHECK(!TestSequenceLocks(CTransaction{tx})); // Sequence locks fail + const int SEQUENCE_LOCK_TIME = 512; // Sequence locks pass 512 seconds later + for (int i = 0; i < CBlockIndex::nMedianTimeSpan; ++i) + m_node.chainman->ActiveChain().Tip()->GetAncestor(m_node.chainman->ActiveChain().Tip()->nHeight - i)->nTime += SEQUENCE_LOCK_TIME; // Trick the MedianTimePast { CBlockIndex* active_chain_tip = m_node.chainman->ActiveChain().Tip(); - BOOST_CHECK(SequenceLocks(CTransaction(tx), flags, prevheights, CreateBlockIndex(active_chain_tip->nHeight + 1, active_chain_tip))); // Sequence locks pass 512 seconds later + BOOST_CHECK(SequenceLocks(CTransaction(tx), flags, prevheights, CreateBlockIndex(active_chain_tip->nHeight + 1, active_chain_tip))); } - for (int i = 0; i < CBlockIndex::nMedianTimeSpan; i++) - m_node.chainman->ActiveChain().Tip()->GetAncestor(m_node.chainman->ActiveChain().Tip()->nHeight - i)->nTime -= 512; //undo tricked MTP + for (int i = 0; i < CBlockIndex::nMedianTimeSpan; ++i) { + CBlockIndex* ancestor{Assert(m_node.chainman->ActiveChain().Tip()->GetAncestor(m_node.chainman->ActiveChain().Tip()->nHeight - i))}; + ancestor->nTime -= SEQUENCE_LOCK_TIME; // undo tricked MTP + } // absolute height locked tx.vin[0].prevout.hash = txFirst[2]->GetHash(); @@ -464,8 +425,8 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.nLockTime = m_node.chainman->ActiveChain().Tip()->nHeight + 1; hash = tx.GetHash(); m_node.mempool->addUnchecked(entry.Time(GetTime()).FromTx(tx)); - BOOST_CHECK(!CheckFinalTx(m_node.chainman->ActiveChain().Tip(), CTransaction(tx), flags)); // Locktime fails - BOOST_CHECK(TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks pass + BOOST_CHECK(!CheckFinalTxAtTip(m_node.chainman->ActiveChain().Tip(), CTransaction{tx})); // Locktime fails + BOOST_CHECK(TestSequenceLocks(CTransaction{tx})); // Sequence locks pass BOOST_CHECK(IsFinalTx(CTransaction(tx), m_node.chainman->ActiveChain().Tip()->nHeight + 2, m_node.chainman->ActiveChain().Tip()->GetMedianTimePast())); // Locktime passes on 2nd block // absolute time locked @@ -475,8 +436,8 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) prevheights[0] = baseheight + 4; hash = tx.GetHash(); m_node.mempool->addUnchecked(entry.Time(GetTime()).FromTx(tx)); - BOOST_CHECK(!CheckFinalTx(m_node.chainman->ActiveChain().Tip(), CTransaction(tx), flags)); // Locktime fails - BOOST_CHECK(TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks pass + BOOST_CHECK(!CheckFinalTxAtTip(m_node.chainman->ActiveChain().Tip(), CTransaction{tx})); // Locktime fails + BOOST_CHECK(TestSequenceLocks(CTransaction{tx})); // Sequence locks pass BOOST_CHECK(IsFinalTx(CTransaction(tx), m_node.chainman->ActiveChain().Tip()->nHeight + 2, m_node.chainman->ActiveChain().Tip()->GetMedianTimePast() + 1)); // Locktime passes 1 second later // mempool-dependent transactions (not added) @@ -484,14 +445,14 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) prevheights[0] = m_node.chainman->ActiveChain().Tip()->nHeight + 1; tx.nLockTime = 0; tx.vin[0].nSequence = 0; - BOOST_CHECK(CheckFinalTx(m_node.chainman->ActiveChain().Tip(), CTransaction(tx), flags)); // Locktime passes - BOOST_CHECK(TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks pass + BOOST_CHECK(CheckFinalTxAtTip(m_node.chainman->ActiveChain().Tip(), CTransaction{tx})); // Locktime passes + BOOST_CHECK(TestSequenceLocks(CTransaction{tx})); // Sequence locks pass tx.vin[0].nSequence = 1; - BOOST_CHECK(!TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks fail + BOOST_CHECK(!TestSequenceLocks(CTransaction{tx})); // Sequence locks fail tx.vin[0].nSequence = CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG; - BOOST_CHECK(TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks pass + BOOST_CHECK(TestSequenceLocks(CTransaction{tx})); // Sequence locks pass tx.vin[0].nSequence = CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG | 1; - BOOST_CHECK(!TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks fail + BOOST_CHECK(!TestSequenceLocks(CTransaction{tx})); // Sequence locks fail BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); @@ -500,14 +461,140 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) // but relative locked txs will if inconsistently added to mempool. // For now these will still generate a valid template until BIP68 soft fork BOOST_CHECK_EQUAL(pblocktemplate->block.vtx.size(), 3U); - // However if we advance height by 1 and time by 512, all of them should be mined - for (int i = 0; i < CBlockIndex::nMedianTimeSpan; i++) - m_node.chainman->ActiveChain().Tip()->GetAncestor(m_node.chainman->ActiveChain().Tip()->nHeight - i)->nTime += 512; //Trick the MedianTimePast + // However if we advance height by 1 and time by SEQUENCE_LOCK_TIME, all of them should be mined + for (int i = 0; i < CBlockIndex::nMedianTimeSpan; ++i) { + CBlockIndex* ancestor{Assert(m_node.chainman->ActiveChain().Tip()->GetAncestor(m_node.chainman->ActiveChain().Tip()->nHeight - i))}; + ancestor->nTime += SEQUENCE_LOCK_TIME; // Trick the MedianTimePast + } m_node.chainman->ActiveChain().Tip()->nHeight++; SetMockTime(m_node.chainman->ActiveChain().Tip()->GetMedianTimePast() + 1); BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); BOOST_CHECK_EQUAL(pblocktemplate->block.vtx.size(), 5U); +} + +void MinerTestingSetup::TestPrioritisedMining(const CChainParams& chainparams, const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst) +{ + TestMemPoolEntryHelper entry; + + // Test that a tx below min fee but prioritised is included + CMutableTransaction tx; + tx.vin.resize(1); + tx.vin[0].prevout.hash = txFirst[0]->GetHash(); + tx.vin[0].prevout.n = 0; + tx.vin[0].scriptSig = CScript() << OP_1; + tx.vout.resize(1); + tx.vout[0].nValue = 5000000000LL; // 0 fee + uint256 hashFreePrioritisedTx = tx.GetHash(); + m_node.mempool->addUnchecked(entry.Fee(0).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + m_node.mempool->PrioritiseTransaction(hashFreePrioritisedTx, 5 * COIN); + + tx.vin[0].prevout.hash = txFirst[1]->GetHash(); + tx.vin[0].prevout.n = 0; + tx.vout[0].nValue = 5000000000LL - 1000; + // This tx has a low fee: 1000 satoshis + uint256 hashParentTx = tx.GetHash(); // save this txid for later use + m_node.mempool->addUnchecked(entry.Fee(1000).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + + // This tx has a medium fee: 10000 satoshis + tx.vin[0].prevout.hash = txFirst[2]->GetHash(); + tx.vout[0].nValue = 5000000000LL - 10000; + uint256 hashMediumFeeTx = tx.GetHash(); + m_node.mempool->addUnchecked(entry.Fee(10000).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + m_node.mempool->PrioritiseTransaction(hashMediumFeeTx, -5 * COIN); + + // This tx also has a low fee, but is prioritised + tx.vin[0].prevout.hash = hashParentTx; + tx.vout[0].nValue = 5000000000LL - 1000 - 1000; // 1000 satoshi fee + uint256 hashPrioritsedChild = tx.GetHash(); + m_node.mempool->addUnchecked(entry.Fee(1000).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); + m_node.mempool->PrioritiseTransaction(hashPrioritsedChild, 2 * COIN); + + // Test that transaction selection properly updates ancestor fee calculations as prioritised + // parents get included in a block. Create a transaction with two prioritised ancestors, each + // included by itself: FreeParent <- FreeChild <- FreeGrandchild. + // When FreeParent is added, a modified entry will be created for FreeChild + FreeGrandchild + // FreeParent's prioritisation should not be included in that entry. + // When FreeChild is included, FreeChild's prioritisation should also not be included. + tx.vin[0].prevout.hash = txFirst[3]->GetHash(); + tx.vout[0].nValue = 5000000000LL; // 0 fee + uint256 hashFreeParent = tx.GetHash(); + m_node.mempool->addUnchecked(entry.Fee(0).SpendsCoinbase(true).FromTx(tx)); + m_node.mempool->PrioritiseTransaction(hashFreeParent, 10 * COIN); + + tx.vin[0].prevout.hash = hashFreeParent; + tx.vout[0].nValue = 5000000000LL; // 0 fee + uint256 hashFreeChild = tx.GetHash(); + m_node.mempool->addUnchecked(entry.Fee(0).SpendsCoinbase(false).FromTx(tx)); + m_node.mempool->PrioritiseTransaction(hashFreeChild, 1 * COIN); + + tx.vin[0].prevout.hash = hashFreeChild; + tx.vout[0].nValue = 5000000000LL; // 0 fee + uint256 hashFreeGrandchild = tx.GetHash(); + m_node.mempool->addUnchecked(entry.Fee(0).SpendsCoinbase(false).FromTx(tx)); + + auto pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); + BOOST_REQUIRE_EQUAL(pblocktemplate->block.vtx.size(), 6U); + BOOST_CHECK(pblocktemplate->block.vtx[1]->GetHash() == hashFreeParent); + BOOST_CHECK(pblocktemplate->block.vtx[2]->GetHash() == hashFreePrioritisedTx); + BOOST_CHECK(pblocktemplate->block.vtx[3]->GetHash() == hashParentTx); + BOOST_CHECK(pblocktemplate->block.vtx[4]->GetHash() == hashPrioritsedChild); + BOOST_CHECK(pblocktemplate->block.vtx[5]->GetHash() == hashFreeChild); + for (size_t i=0; i<pblocktemplate->block.vtx.size(); ++i) { + // The FreeParent and FreeChild's prioritisations should not impact the child. + BOOST_CHECK(pblocktemplate->block.vtx[i]->GetHash() != hashFreeGrandchild); + // De-prioritised transaction should not be included. + BOOST_CHECK(pblocktemplate->block.vtx[i]->GetHash() != hashMediumFeeTx); + } +} + +// NOTE: These tests rely on CreateNewBlock doing its own self-validation! +BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) +{ + // Note that by default, these tests run with size accounting enabled. + const auto chainParams = CreateChainParams(*m_node.args, CBaseChainParams::MAIN); + const CChainParams& chainparams = *chainParams; + CScript scriptPubKey = CScript() << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f") << OP_CHECKSIG; + std::unique_ptr<CBlockTemplate> pblocktemplate; + + fCheckpointsEnabled = false; + + // Simple block creation, nothing special yet: + BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); + + // We can't make transactions until we have inputs + // Therefore, load 110 blocks :) + static_assert(std::size(BLOCKINFO) == 110, "Should have 110 blocks to import"); + int baseheight = 0; + std::vector<CTransactionRef> txFirst; + for (const auto& bi : BLOCKINFO) { + CBlock *pblock = &pblocktemplate->block; // pointer for convenience + { + LOCK(cs_main); + pblock->nVersion = VERSIONBITS_TOP_BITS; + pblock->nTime = m_node.chainman->ActiveChain().Tip()->GetMedianTimePast()+1; + CMutableTransaction txCoinbase(*pblock->vtx[0]); + txCoinbase.nVersion = 1; + txCoinbase.vin[0].scriptSig = CScript{} << (m_node.chainman->ActiveChain().Height() + 1) << bi.extranonce; + txCoinbase.vout.resize(1); // Ignore the (optional) segwit commitment added by CreateNewBlock (as the hardcoded nonces don't account for this) + txCoinbase.vout[0].scriptPubKey = CScript(); + pblock->vtx[0] = MakeTransactionRef(std::move(txCoinbase)); + if (txFirst.size() == 0) + baseheight = m_node.chainman->ActiveChain().Height(); + if (txFirst.size() < 4) + txFirst.push_back(pblock->vtx[0]); + pblock->hashMerkleRoot = BlockMerkleRoot(*pblock); + pblock->nNonce = bi.nonce; + } + std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(*pblock); + BOOST_CHECK(Assert(m_node.chainman)->ProcessNewBlock(chainparams, shared_pblock, true, nullptr)); + pblock->hashPrevBlock = pblock->GetHash(); + } + + LOCK(cs_main); + LOCK(m_node.mempool->cs); + + TestBasicMining(chainparams, scriptPubKey, txFirst, baseheight); m_node.chainman->ActiveChain().Tip()->nHeight--; SetMockTime(0); @@ -515,6 +602,12 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) TestPackageSelection(chainparams, scriptPubKey, txFirst); + m_node.chainman->ActiveChain().Tip()->nHeight--; + SetMockTime(0); + m_node.mempool->clear(); + + TestPrioritisedMining(chainparams, scriptPubKey, txFirst); + fCheckpointsEnabled = true; } diff --git a/src/test/miniscript_tests.cpp b/src/test/miniscript_tests.cpp new file mode 100644 index 0000000000..930582ea24 --- /dev/null +++ b/src/test/miniscript_tests.cpp @@ -0,0 +1,293 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + + +#include <string> + +#include <test/util/setup_common.h> +#include <boost/test/unit_test.hpp> + +#include <hash.h> +#include <pubkey.h> +#include <uint256.h> +#include <crypto/ripemd160.h> +#include <crypto/sha256.h> +#include <script/miniscript.h> + +namespace { + +/** TestData groups various kinds of precomputed data necessary in this test. */ +struct TestData { + //! The only public keys used in this test. + std::vector<CPubKey> pubkeys; + //! A map from the public keys to their CKeyIDs (faster than hashing every time). + std::map<CPubKey, CKeyID> pkhashes; + std::map<CKeyID, CPubKey> pkmap; + + // Various precomputed hashes + std::vector<std::vector<unsigned char>> sha256; + std::vector<std::vector<unsigned char>> ripemd160; + std::vector<std::vector<unsigned char>> hash256; + std::vector<std::vector<unsigned char>> hash160; + + TestData() + { + // We generate 255 public keys and 255 hashes of each type. + for (int i = 1; i <= 255; ++i) { + // This 32-byte array functions as both private key data and hash preimage (31 zero bytes plus any nonzero byte). + unsigned char keydata[32] = {0}; + keydata[31] = i; + + // Compute CPubkey and CKeyID + CKey key; + key.Set(keydata, keydata + 32, true); + CPubKey pubkey = key.GetPubKey(); + CKeyID keyid = pubkey.GetID(); + pubkeys.push_back(pubkey); + pkhashes.emplace(pubkey, keyid); + pkmap.emplace(keyid, pubkey); + + // Compute various hashes + std::vector<unsigned char> hash; + hash.resize(32); + CSHA256().Write(keydata, 32).Finalize(hash.data()); + sha256.push_back(hash); + CHash256().Write(keydata).Finalize(hash); + hash256.push_back(hash); + hash.resize(20); + CRIPEMD160().Write(keydata, 32).Finalize(hash.data()); + ripemd160.push_back(hash); + CHash160().Write(keydata).Finalize(hash); + hash160.push_back(hash); + } + } +}; + +//! Global TestData object +std::unique_ptr<const TestData> g_testdata; + +/** A class encapsulating conversion routing for CPubKey. */ +struct KeyConverter { + typedef CPubKey Key; + + //! Convert a public key to bytes. + std::vector<unsigned char> ToPKBytes(const CPubKey& key) const { return {key.begin(), key.end()}; } + + //! Convert a public key to its Hash160 bytes (precomputed). + std::vector<unsigned char> ToPKHBytes(const CPubKey& key) const + { + auto it = g_testdata->pkhashes.find(key); + assert(it != g_testdata->pkhashes.end()); + return {it->second.begin(), it->second.end()}; + } + + //! Parse a public key from a range of hex characters. + template<typename I> + bool FromString(I first, I last, CPubKey& key) const { + auto bytes = ParseHex(std::string(first, last)); + key.Set(bytes.begin(), bytes.end()); + return key.IsValid(); + } + + template<typename I> + bool FromPKBytes(I first, I last, CPubKey& key) const { + key.Set(first, last); + return key.IsValid(); + } + + template<typename I> + bool FromPKHBytes(I first, I last, CPubKey& key) const { + assert(last - first == 20); + CKeyID keyid; + std::copy(first, last, keyid.begin()); + auto it = g_testdata->pkmap.find(keyid); + assert(it != g_testdata->pkmap.end()); + key = it->second; + return true; + } +}; + +//! Singleton instance of KeyConverter. +const KeyConverter CONVERTER{}; + +using miniscript::operator"" _mst; + +enum TestMode : int { + TESTMODE_INVALID = 0, + TESTMODE_VALID = 1, + TESTMODE_NONMAL = 2, + TESTMODE_NEEDSIG = 4, + TESTMODE_TIMELOCKMIX = 8 +}; + +void Test(const std::string& ms, const std::string& hexscript, int mode, int opslimit = -1, int stacklimit = -1) +{ + auto node = miniscript::FromString(ms, CONVERTER); + if (mode == TESTMODE_INVALID) { + BOOST_CHECK_MESSAGE(!node || !node->IsValid(), "Unexpectedly valid: " + ms); + } else { + BOOST_CHECK_MESSAGE(node, "Unparseable: " + ms); + BOOST_CHECK_MESSAGE(node->IsValid(), "Invalid: " + ms); + BOOST_CHECK_MESSAGE(node->IsValidTopLevel(), "Invalid top level: " + ms); + auto computed_script = node->ToScript(CONVERTER); + BOOST_CHECK_MESSAGE(node->ScriptSize() == computed_script.size(), "Script size mismatch: " + ms); + if (hexscript != "?") BOOST_CHECK_MESSAGE(HexStr(computed_script) == hexscript, "Script mismatch: " + ms + " (" + HexStr(computed_script) + " vs " + hexscript + ")"); + BOOST_CHECK_MESSAGE(node->IsNonMalleable() == !!(mode & TESTMODE_NONMAL), "Malleability mismatch: " + ms); + BOOST_CHECK_MESSAGE(node->NeedsSignature() == !!(mode & TESTMODE_NEEDSIG), "Signature necessity mismatch: " + ms); + BOOST_CHECK_MESSAGE((node->GetType() << "k"_mst) == !(mode & TESTMODE_TIMELOCKMIX), "Timelock mix mismatch: " + ms); + auto inferred_miniscript = miniscript::FromScript(computed_script, CONVERTER); + BOOST_CHECK_MESSAGE(inferred_miniscript, "Cannot infer miniscript from script: " + ms); + BOOST_CHECK_MESSAGE(inferred_miniscript->ToScript(CONVERTER) == computed_script, "Roundtrip failure: miniscript->script != miniscript->script->miniscript->script: " + ms); + if (opslimit != -1) BOOST_CHECK_MESSAGE((int)node->GetOps() == opslimit, "Ops limit mismatch: " << ms << " (" << node->GetOps() << " vs " << opslimit << ")"); + if (stacklimit != -1) BOOST_CHECK_MESSAGE((int)node->GetStackSize() == stacklimit, "Stack limit mismatch: " << ms << " (" << node->GetStackSize() << " vs " << stacklimit << ")"); + } +} +} // namespace + +BOOST_FIXTURE_TEST_SUITE(miniscript_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(fixed_tests) +{ + g_testdata.reset(new TestData()); + + // Validity rules + Test("l:older(1)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // older(1): valid + Test("l:older(0)", "?", TESTMODE_INVALID); // older(0): k must be at least 1 + Test("l:older(2147483647)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // older(2147483647): valid + Test("l:older(2147483648)", "?", TESTMODE_INVALID); // older(2147483648): k must be below 2^31 + Test("u:after(1)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // after(1): valid + Test("u:after(0)", "?", TESTMODE_INVALID); // after(0): k must be at least 1 + Test("u:after(2147483647)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // after(2147483647): valid + Test("u:after(2147483648)", "?", TESTMODE_INVALID); // after(2147483648): k must be below 2^31 + Test("andor(0,1,1)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // andor(Bdu,B,B): valid + Test("andor(a:0,1,1)", "?", TESTMODE_INVALID); // andor(Wdu,B,B): X must be B + Test("andor(0,a:1,a:1)", "?", TESTMODE_INVALID); // andor(Bdu,W,W): Y and Z must be B/V/K + Test("andor(1,1,1)", "?", TESTMODE_INVALID); // andor(Bu,B,B): X must be d + Test("andor(n:or_i(0,after(1)),1,1)", "?", TESTMODE_VALID); // andor(Bdu,B,B): valid + Test("andor(or_i(0,after(1)),1,1)", "?", TESTMODE_INVALID); // andor(Bd,B,B): X must be u + Test("c:andor(0,pk_k(03a0434d9e47f3c86235477c7b1ae6ae5d3442d49b1943c2b752a68e2a47e247c7),pk_k(036d2b085e9e382ed10b69fc311a03f8641ccfff21574de0927513a49d9a688a00))", "?", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG); // andor(Bdu,K,K): valid + Test("t:andor(0,v:1,v:1)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // andor(Bdu,V,V): valid + Test("and_v(v:1,1)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // and_v(V,B): valid + Test("t:and_v(v:1,v:1)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // and_v(V,V): valid + Test("c:and_v(v:1,pk_k(036d2b085e9e382ed10b69fc311a03f8641ccfff21574de0927513a49d9a688a00))", "?", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG); // and_v(V,K): valid + Test("and_v(1,1)", "?", TESTMODE_INVALID); // and_v(B,B): X must be V + Test("and_v(pk_k(02352bbf4a4cdd12564f93fa332ce333301d9ad40271f8107181340aef25be59d5),1)", "?", TESTMODE_INVALID); // and_v(K,B): X must be V + Test("and_v(v:1,a:1)", "?", TESTMODE_INVALID); // and_v(K,W): Y must be B/V/K + Test("and_b(1,a:1)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // and_b(B,W): valid + Test("and_b(1,1)", "?", TESTMODE_INVALID); // and_b(B,B): Y must W + Test("and_b(v:1,a:1)", "?", TESTMODE_INVALID); // and_b(V,W): X must be B + Test("and_b(a:1,a:1)", "?", TESTMODE_INVALID); // and_b(W,W): X must be B + Test("and_b(pk_k(025601570cb47f238d2b0286db4a990fa0f3ba28d1a319f5e7cf55c2a2444da7cc),a:1)", "?", TESTMODE_INVALID); // and_b(K,W): X must be B + Test("or_b(0,a:0)", "?", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG); // or_b(Bd,Wd): valid + Test("or_b(1,a:0)", "?", TESTMODE_INVALID); // or_b(B,Wd): X must be d + Test("or_b(0,a:1)", "?", TESTMODE_INVALID); // or_b(Bd,W): Y must be d + Test("or_b(0,0)", "?", TESTMODE_INVALID); // or_b(Bd,Bd): Y must W + Test("or_b(v:0,a:0)", "?", TESTMODE_INVALID); // or_b(V,Wd): X must be B + Test("or_b(a:0,a:0)", "?", TESTMODE_INVALID); // or_b(Wd,Wd): X must be B + Test("or_b(pk_k(025601570cb47f238d2b0286db4a990fa0f3ba28d1a319f5e7cf55c2a2444da7cc),a:0)", "?", TESTMODE_INVALID); // or_b(Kd,Wd): X must be B + Test("t:or_c(0,v:1)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // or_c(Bdu,V): valid + Test("t:or_c(a:0,v:1)", "?", TESTMODE_INVALID); // or_c(Wdu,V): X must be B + Test("t:or_c(1,v:1)", "?", TESTMODE_INVALID); // or_c(Bu,V): X must be d + Test("t:or_c(n:or_i(0,after(1)),v:1)", "?", TESTMODE_VALID); // or_c(Bdu,V): valid + Test("t:or_c(or_i(0,after(1)),v:1)", "?", TESTMODE_INVALID); // or_c(Bd,V): X must be u + Test("t:or_c(0,1)", "?", TESTMODE_INVALID); // or_c(Bdu,B): Y must be V + Test("or_d(0,1)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // or_d(Bdu,B): valid + Test("or_d(a:0,1)", "?", TESTMODE_INVALID); // or_d(Wdu,B): X must be B + Test("or_d(1,1)", "?", TESTMODE_INVALID); // or_d(Bu,B): X must be d + Test("or_d(n:or_i(0,after(1)),1)", "?", TESTMODE_VALID); // or_d(Bdu,B): valid + Test("or_d(or_i(0,after(1)),1)", "?", TESTMODE_INVALID); // or_d(Bd,B): X must be u + Test("or_d(0,v:1)", "?", TESTMODE_INVALID); // or_d(Bdu,V): Y must be B + Test("or_i(1,1)", "?", TESTMODE_VALID); // or_i(B,B): valid + Test("t:or_i(v:1,v:1)", "?", TESTMODE_VALID); // or_i(V,V): valid + Test("c:or_i(pk_k(03a0434d9e47f3c86235477c7b1ae6ae5d3442d49b1943c2b752a68e2a47e247c7),pk_k(036d2b085e9e382ed10b69fc311a03f8641ccfff21574de0927513a49d9a688a00))", "?", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG); // or_i(K,K): valid + Test("or_i(a:1,a:1)", "?", TESTMODE_INVALID); // or_i(W,W): X and Y must be B/V/K + Test("or_b(l:after(100),al:after(1000000000))", "?", TESTMODE_VALID); // or_b(timelock, heighlock) valid + Test("and_b(after(100),a:after(1000000000))", "?", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_TIMELOCKMIX); // and_b(timelock, heighlock) invalid + Test("pk(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65)", "2103d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65ac", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG); // alias to c:pk_k + Test("pkh(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65)", "76a914fcd35ddacad9f2d5be5e464639441c6065e6955d88ac", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG); // alias to c:pk_h + + + // Randomly generated test set that covers the majority of type and node type combinations + Test("lltvln:after(1231488000)", "6300676300676300670400046749b1926869516868", TESTMODE_VALID | TESTMODE_NONMAL, 12, 4); + Test("uuj:and_v(v:multi(2,03d01115d548e7561b15c38f004d734633687cf4419620095bc5b0f47070afe85a,025601570cb47f238d2b0286db4a990fa0f3ba28d1a319f5e7cf55c2a2444da7cc),after(1231488000))", "6363829263522103d01115d548e7561b15c38f004d734633687cf4419620095bc5b0f47070afe85a21025601570cb47f238d2b0286db4a990fa0f3ba28d1a319f5e7cf55c2a2444da7cc52af0400046749b168670068670068", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG, 14, 6); + Test("or_b(un:multi(2,03daed4f2be3a8bf278e70132fb0beb7522f570e144bf615c07e996d443dee8729,024ce119c96e2fa357200b559b2f7dd5a5f02d5290aff74b03f3e471b273211c97),al:older(16))", "63522103daed4f2be3a8bf278e70132fb0beb7522f570e144bf615c07e996d443dee872921024ce119c96e2fa357200b559b2f7dd5a5f02d5290aff74b03f3e471b273211c9752ae926700686b63006760b2686c9b", TESTMODE_VALID, 14, 6); + Test("j:and_v(vdv:after(1567547623),older(2016))", "829263766304e7e06e5db169686902e007b268", TESTMODE_VALID | TESTMODE_NONMAL, 11, 2); + Test("t:and_v(vu:hash256(131772552c01444cd81360818376a040b7c3b2b7b0a53550ee3edde216cec61b),v:sha256(ec4916dd28fc4c10d78e287ca5d9cc51ee1ae73cbfde08c6b37324cbfaac8bc5))", "6382012088aa20131772552c01444cd81360818376a040b7c3b2b7b0a53550ee3edde216cec61b876700686982012088a820ec4916dd28fc4c10d78e287ca5d9cc51ee1ae73cbfde08c6b37324cbfaac8bc58851", TESTMODE_VALID | TESTMODE_NONMAL, 12, 4); + Test("t:andor(multi(3,02d7924d4f7d43ea965a465ae3095ff41131e5946f3c85f79e44adbcf8e27e080e,03fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556,02e493dbf1c10d80f3581e4904930b1404cc6c13900ee0758474fa94abe8c4cd13),v:older(4194305),v:sha256(9267d3dbed802941483f1afa2a6bc68de5f653128aca9bf1461c5d0a3ad36ed2))", "532102d7924d4f7d43ea965a465ae3095ff41131e5946f3c85f79e44adbcf8e27e080e2103fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a14602975562102e493dbf1c10d80f3581e4904930b1404cc6c13900ee0758474fa94abe8c4cd1353ae6482012088a8209267d3dbed802941483f1afa2a6bc68de5f653128aca9bf1461c5d0a3ad36ed2886703010040b2696851", TESTMODE_VALID | TESTMODE_NONMAL, 13, 6); + Test("or_d(multi(1,02f9308a019258c31049344f85f89d5229b531c845836f99b08601f113bce036f9),or_b(multi(3,022f01e5e15cca351daff3843fb70f3c2f0a1bdd05e5af888a67784ef3e10a2a01,032fa2104d6b38d11b0230010559879124e42ab8dfeff5ff29dc9cdadd4ecacc3f,03d01115d548e7561b15c38f004d734633687cf4419620095bc5b0f47070afe85a),su:after(500000)))", "512102f9308a019258c31049344f85f89d5229b531c845836f99b08601f113bce036f951ae73645321022f01e5e15cca351daff3843fb70f3c2f0a1bdd05e5af888a67784ef3e10a2a0121032fa2104d6b38d11b0230010559879124e42ab8dfeff5ff29dc9cdadd4ecacc3f2103d01115d548e7561b15c38f004d734633687cf4419620095bc5b0f47070afe85a53ae7c630320a107b16700689b68", TESTMODE_VALID | TESTMODE_NONMAL, 15, 8); + Test("or_d(sha256(38df1c1f64a24a77b23393bca50dff872e31edc4f3b5aa3b90ad0b82f4f089b6),and_n(un:after(499999999),older(4194305)))", "82012088a82038df1c1f64a24a77b23393bca50dff872e31edc4f3b5aa3b90ad0b82f4f089b68773646304ff64cd1db19267006864006703010040b26868", TESTMODE_VALID, 16, 2); + Test("and_v(or_i(v:multi(2,02c6047f9441ed7d6d3045406e95c07cd85c778e4b8cef3ca7abac09b95c709ee5,03774ae7f858a9411e5ef4246b70c65aac5649980be5c17891bbec17895da008cb),v:multi(2,03e60fce93b59e9ec53011aabc21c23e97b2a31369b87a5ae9c44ee89e2a6dec0a,025cbdf0646e5db4eaa398f365f2ea7a0e3d419b7e0330e39ce92bddedcac4f9bc)),sha256(d1ec675902ef1633427ca360b290b0b3045a0d9058ddb5e648b4c3c3224c5c68))", "63522102c6047f9441ed7d6d3045406e95c07cd85c778e4b8cef3ca7abac09b95c709ee52103774ae7f858a9411e5ef4246b70c65aac5649980be5c17891bbec17895da008cb52af67522103e60fce93b59e9ec53011aabc21c23e97b2a31369b87a5ae9c44ee89e2a6dec0a21025cbdf0646e5db4eaa398f365f2ea7a0e3d419b7e0330e39ce92bddedcac4f9bc52af6882012088a820d1ec675902ef1633427ca360b290b0b3045a0d9058ddb5e648b4c3c3224c5c6887", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG, 11, 6); + Test("j:and_b(multi(2,0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798,024ce119c96e2fa357200b559b2f7dd5a5f02d5290aff74b03f3e471b273211c97),s:or_i(older(1),older(4252898)))", "82926352210279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f8179821024ce119c96e2fa357200b559b2f7dd5a5f02d5290aff74b03f3e471b273211c9752ae7c6351b26703e2e440b2689a68", TESTMODE_VALID | TESTMODE_NEEDSIG, 14, 5); + Test("and_b(older(16),s:or_d(sha256(e38990d0c7fc009880a9c07c23842e886c6bbdc964ce6bdd5817ad357335ee6f),n:after(1567547623)))", "60b27c82012088a820e38990d0c7fc009880a9c07c23842e886c6bbdc964ce6bdd5817ad357335ee6f87736404e7e06e5db192689a", TESTMODE_VALID, 12, 2); + Test("j:and_v(v:hash160(20195b5a3d650c17f0f29f91c33f8f6335193d07),or_d(sha256(96de8fc8c256fa1e1556d41af431cace7dca68707c78dd88c3acab8b17164c47),older(16)))", "82926382012088a91420195b5a3d650c17f0f29f91c33f8f6335193d078882012088a82096de8fc8c256fa1e1556d41af431cace7dca68707c78dd88c3acab8b17164c4787736460b26868", TESTMODE_VALID, 16, 3); + Test("and_b(hash256(32ba476771d01e37807990ead8719f08af494723de1d228f2c2c07cc0aa40bac),a:and_b(hash256(131772552c01444cd81360818376a040b7c3b2b7b0a53550ee3edde216cec61b),a:older(1)))", "82012088aa2032ba476771d01e37807990ead8719f08af494723de1d228f2c2c07cc0aa40bac876b82012088aa20131772552c01444cd81360818376a040b7c3b2b7b0a53550ee3edde216cec61b876b51b26c9a6c9a", TESTMODE_VALID | TESTMODE_NONMAL, 15, 3); + Test("thresh(2,multi(2,03a0434d9e47f3c86235477c7b1ae6ae5d3442d49b1943c2b752a68e2a47e247c7,036d2b085e9e382ed10b69fc311a03f8641ccfff21574de0927513a49d9a688a00),a:multi(1,036d2b085e9e382ed10b69fc311a03f8641ccfff21574de0927513a49d9a688a00),ac:pk_k(022f01e5e15cca351daff3843fb70f3c2f0a1bdd05e5af888a67784ef3e10a2a01))", "522103a0434d9e47f3c86235477c7b1ae6ae5d3442d49b1943c2b752a68e2a47e247c721036d2b085e9e382ed10b69fc311a03f8641ccfff21574de0927513a49d9a688a0052ae6b5121036d2b085e9e382ed10b69fc311a03f8641ccfff21574de0927513a49d9a688a0051ae6c936b21022f01e5e15cca351daff3843fb70f3c2f0a1bdd05e5af888a67784ef3e10a2a01ac6c935287", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG, 13, 7); + Test("and_n(sha256(d1ec675902ef1633427ca360b290b0b3045a0d9058ddb5e648b4c3c3224c5c68),t:or_i(v:older(4252898),v:older(144)))", "82012088a820d1ec675902ef1633427ca360b290b0b3045a0d9058ddb5e648b4c3c3224c5c68876400676303e2e440b26967029000b269685168", TESTMODE_VALID, 14, 3); + Test("or_d(nd:and_v(v:older(4252898),v:older(4252898)),sha256(38df1c1f64a24a77b23393bca50dff872e31edc4f3b5aa3b90ad0b82f4f089b6))", "766303e2e440b26903e2e440b2696892736482012088a82038df1c1f64a24a77b23393bca50dff872e31edc4f3b5aa3b90ad0b82f4f089b68768", TESTMODE_VALID, 15, 3); + Test("c:and_v(or_c(sha256(9267d3dbed802941483f1afa2a6bc68de5f653128aca9bf1461c5d0a3ad36ed2),v:multi(1,02c44d12c7065d812e8acf28d7cbb19f9011ecd9e9fdf281b0e6a3b5e87d22e7db)),pk_k(03acd484e2f0c7f65309ad178a9f559abde09796974c57e714c35f110dfc27ccbe))", "82012088a8209267d3dbed802941483f1afa2a6bc68de5f653128aca9bf1461c5d0a3ad36ed28764512102c44d12c7065d812e8acf28d7cbb19f9011ecd9e9fdf281b0e6a3b5e87d22e7db51af682103acd484e2f0c7f65309ad178a9f559abde09796974c57e714c35f110dfc27ccbeac", TESTMODE_VALID | TESTMODE_NEEDSIG, 8, 3); + Test("c:and_v(or_c(multi(2,036d2b085e9e382ed10b69fc311a03f8641ccfff21574de0927513a49d9a688a00,02352bbf4a4cdd12564f93fa332ce333301d9ad40271f8107181340aef25be59d5),v:ripemd160(1b0f3c404d12075c68c938f9f60ebea4f74941a0)),pk_k(03fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556))", "5221036d2b085e9e382ed10b69fc311a03f8641ccfff21574de0927513a49d9a688a002102352bbf4a4cdd12564f93fa332ce333301d9ad40271f8107181340aef25be59d552ae6482012088a6141b0f3c404d12075c68c938f9f60ebea4f74941a088682103fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556ac", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG, 10, 6); + Test("and_v(andor(hash256(8a35d9ca92a48eaade6f53a64985e9e2afeb74dcf8acb4c3721e0dc7e4294b25),v:hash256(939894f70e6c3a25da75da0cc2071b4076d9b006563cf635986ada2e93c0d735),v:older(50000)),after(499999999))", "82012088aa208a35d9ca92a48eaade6f53a64985e9e2afeb74dcf8acb4c3721e0dc7e4294b2587640350c300b2696782012088aa20939894f70e6c3a25da75da0cc2071b4076d9b006563cf635986ada2e93c0d735886804ff64cd1db1", TESTMODE_VALID, 14, 3); + Test("andor(hash256(5f8d30e655a7ba0d7596bb3ddfb1d2d20390d23b1845000e1e118b3be1b3f040),j:and_v(v:hash160(3a2bff0da9d96868e66abc4427bea4691cf61ccd),older(4194305)),ripemd160(44d90e2d3714c8663b632fcf0f9d5f22192cc4c8))", "82012088aa205f8d30e655a7ba0d7596bb3ddfb1d2d20390d23b1845000e1e118b3be1b3f040876482012088a61444d90e2d3714c8663b632fcf0f9d5f22192cc4c8876782926382012088a9143a2bff0da9d96868e66abc4427bea4691cf61ccd8803010040b26868", TESTMODE_VALID, 20, 3); + Test("or_i(c:and_v(v:after(500000),pk_k(02c6047f9441ed7d6d3045406e95c07cd85c778e4b8cef3ca7abac09b95c709ee5)),sha256(d9147961436944f43cd99d28b2bbddbf452ef872b30c8279e255e7daafc7f946))", "630320a107b1692102c6047f9441ed7d6d3045406e95c07cd85c778e4b8cef3ca7abac09b95c709ee5ac6782012088a820d9147961436944f43cd99d28b2bbddbf452ef872b30c8279e255e7daafc7f9468768", TESTMODE_VALID | TESTMODE_NONMAL, 10, 3); + Test("thresh(2,c:pk_h(025cbdf0646e5db4eaa398f365f2ea7a0e3d419b7e0330e39ce92bddedcac4f9bc),s:sha256(e38990d0c7fc009880a9c07c23842e886c6bbdc964ce6bdd5817ad357335ee6f),a:hash160(dd69735817e0e3f6f826a9238dc2e291184f0131))", "76a9145dedfbf9ea599dd4e3ca6a80b333c472fd0b3f6988ac7c82012088a820e38990d0c7fc009880a9c07c23842e886c6bbdc964ce6bdd5817ad357335ee6f87936b82012088a914dd69735817e0e3f6f826a9238dc2e291184f0131876c935287", TESTMODE_VALID, 18, 5); + Test("and_n(sha256(9267d3dbed802941483f1afa2a6bc68de5f653128aca9bf1461c5d0a3ad36ed2),uc:and_v(v:older(144),pk_k(03fe72c435413d33d48ac09c9161ba8b09683215439d62b7940502bda8b202e6ce)))", "82012088a8209267d3dbed802941483f1afa2a6bc68de5f653128aca9bf1461c5d0a3ad36ed28764006763029000b2692103fe72c435413d33d48ac09c9161ba8b09683215439d62b7940502bda8b202e6ceac67006868", TESTMODE_VALID | TESTMODE_NEEDSIG, 13, 4); + Test("and_n(c:pk_k(03daed4f2be3a8bf278e70132fb0beb7522f570e144bf615c07e996d443dee8729),and_b(l:older(4252898),a:older(16)))", "2103daed4f2be3a8bf278e70132fb0beb7522f570e144bf615c07e996d443dee8729ac64006763006703e2e440b2686b60b26c9a68", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG | TESTMODE_TIMELOCKMIX, 12, 3); + Test("c:or_i(and_v(v:older(16),pk_h(02d7924d4f7d43ea965a465ae3095ff41131e5946f3c85f79e44adbcf8e27e080e)),pk_h(026a245bf6dc698504c89a20cfded60853152b695336c28063b61c65cbd269e6b4))", "6360b26976a9149fc5dbe5efdce10374a4dd4053c93af540211718886776a9142fbd32c8dd59ee7c17e66cb6ebea7e9846c3040f8868ac", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG, 12, 4); + Test("or_d(c:pk_h(02e493dbf1c10d80f3581e4904930b1404cc6c13900ee0758474fa94abe8c4cd13),andor(c:pk_k(024ce119c96e2fa357200b559b2f7dd5a5f02d5290aff74b03f3e471b273211c97),older(2016),after(1567547623)))", "76a914c42e7ef92fdb603af844d064faad95db9bcdfd3d88ac736421024ce119c96e2fa357200b559b2f7dd5a5f02d5290aff74b03f3e471b273211c97ac6404e7e06e5db16702e007b26868", TESTMODE_VALID | TESTMODE_NONMAL, 13, 4); + Test("c:andor(ripemd160(6ad07d21fd5dfc646f0b30577045ce201616b9ba),pk_h(02d7924d4f7d43ea965a465ae3095ff41131e5946f3c85f79e44adbcf8e27e080e),and_v(v:hash256(8a35d9ca92a48eaade6f53a64985e9e2afeb74dcf8acb4c3721e0dc7e4294b25),pk_h(03d01115d548e7561b15c38f004d734633687cf4419620095bc5b0f47070afe85a)))", "82012088a6146ad07d21fd5dfc646f0b30577045ce201616b9ba876482012088aa208a35d9ca92a48eaade6f53a64985e9e2afeb74dcf8acb4c3721e0dc7e4294b258876a914dd100be7d9aea5721158ebde6d6a1fd8fff93bb1886776a9149fc5dbe5efdce10374a4dd4053c93af5402117188868ac", TESTMODE_VALID | TESTMODE_NEEDSIG, 18, 4); + Test("c:andor(u:ripemd160(6ad07d21fd5dfc646f0b30577045ce201616b9ba),pk_h(03daed4f2be3a8bf278e70132fb0beb7522f570e144bf615c07e996d443dee8729),or_i(pk_h(022f01e5e15cca351daff3843fb70f3c2f0a1bdd05e5af888a67784ef3e10a2a01),pk_h(0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798)))", "6382012088a6146ad07d21fd5dfc646f0b30577045ce201616b9ba87670068646376a9149652d86bedf43ad264362e6e6eba6eb764508127886776a914751e76e8199196d454941c45d1b3a323f1433bd688686776a91420d637c1a6404d2227f3561fdbaff5a680dba6488868ac", TESTMODE_VALID | TESTMODE_NEEDSIG, 23, 5); + Test("c:or_i(andor(c:pk_h(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),pk_h(022f01e5e15cca351daff3843fb70f3c2f0a1bdd05e5af888a67784ef3e10a2a01),pk_h(02c6047f9441ed7d6d3045406e95c07cd85c778e4b8cef3ca7abac09b95c709ee5)),pk_k(02d7924d4f7d43ea965a465ae3095ff41131e5946f3c85f79e44adbcf8e27e080e))", "6376a914fcd35ddacad9f2d5be5e464639441c6065e6955d88ac6476a91406afd46bcdfd22ef94ac122aa11f241244a37ecc886776a9149652d86bedf43ad264362e6e6eba6eb7645081278868672102d7924d4f7d43ea965a465ae3095ff41131e5946f3c85f79e44adbcf8e27e080e68ac", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG, 17, 6); + Test("thresh(1,c:pk_k(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),altv:after(1000000000),altv:after(100))", "2103d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65ac6b6300670400ca9a3bb16951686c936b6300670164b16951686c935187", TESTMODE_VALID, 18, 4); + Test("thresh(2,c:pk_k(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),ac:pk_k(03fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556),altv:after(1000000000),altv:after(100))", "2103d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65ac6b2103fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556ac6c936b6300670400ca9a3bb16951686c936b6300670164b16951686c935287", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_TIMELOCKMIX, 22, 5); + + // Misc unit tests + // A Script with a non minimal push is invalid + std::vector<unsigned char> nonminpush = ParseHex("0000210232780000feff00ffffffffffff21ff005f00ae21ae00000000060602060406564c2102320000060900fe00005f00ae21ae00100000060606060606000000000000000000000000000000000000000000000000000000000000000000"); + const CScript nonminpush_script(nonminpush.begin(), nonminpush.end()); + BOOST_CHECK(miniscript::FromScript(nonminpush_script, CONVERTER) == nullptr); + // A non-minimal VERIFY (<key> CHECKSIG VERIFY 1) + std::vector<unsigned char> nonminverify = ParseHex("2103a0434d9e47f3c86235477c7b1ae6ae5d3442d49b1943c2b752a68e2a47e247c7ac6951"); + const CScript nonminverify_script(nonminverify.begin(), nonminverify.end()); + BOOST_CHECK(miniscript::FromScript(nonminverify_script, CONVERTER) == nullptr); + // A threshold as large as the number of subs is valid. + Test("thresh(2,c:pk_k(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),altv:after(100))", "2103d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65ac6b6300670164b16951686c935287", TESTMODE_VALID | TESTMODE_NEEDSIG | TESTMODE_NONMAL); + // A threshold of 1 is valid. + Test("thresh(1,c:pk_k(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),sc:pk_k(03fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556))", "2103d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65ac7c2103fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556ac935187", TESTMODE_VALID | TESTMODE_NEEDSIG | TESTMODE_NONMAL); + // A threshold with a k larger than the number of subs is invalid + Test("thresh(3,c:pk_k(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),sc:pk_k(03fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556))", "2103d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65ac7c2103fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556ac935187", TESTMODE_INVALID); + // A threshold with a k null is invalid + Test("thresh(0,c:pk_k(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),sc:pk_k(03fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556))", "2103d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65ac7c2103fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556ac935187", TESTMODE_INVALID); + // For CHECKMULTISIG the OP cost is the number of keys, but the stack size is the number of sigs (+1) + const auto ms_multi = miniscript::FromString("multi(1,03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65,03fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556,0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798)", CONVERTER); + BOOST_CHECK(ms_multi); + BOOST_CHECK_EQUAL(ms_multi->GetOps(), 4); // 3 pubkeys + CMS + BOOST_CHECK_EQUAL(ms_multi->GetStackSize(), 3); // 1 sig + dummy elem + script push + // The 'd:' wrapper leaves on the stack what was DUP'ed at the beginning of its execution. + // Since it contains an OP_IF just after on the same element, we can make sure that the element + // in question must be OP_1 if OP_IF enforces that its argument must only be OP_1 or the empty + // vector (since otherwise the execution would immediately fail). This is the MINIMALIF rule. + // Unfortunately, this rule is consensus for Taproot but only policy for P2WSH. Therefore we can't + // (for now) have 'd:' be 'u'. This tests we can't use a 'd:' wrapper for a thresh, which requires + // its subs to all be 'u' (taken from https://github.com/rust-bitcoin/rust-miniscript/discussions/341). + const auto ms_minimalif = miniscript::FromString("thresh(3,c:pk_k(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),sc:pk_k(03fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556),sc:pk_k(0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798),sdv:older(32))", CONVERTER); + BOOST_CHECK(!ms_minimalif); + + // Timelock tests + Test("after(100)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // only heightlock + Test("after(1000000000)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // only timelock + Test("or_b(l:after(100),al:after(1000000000))", "?", TESTMODE_VALID); // or_b(timelock, heighlock) valid + Test("and_b(after(100),a:after(1000000000))", "?", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_TIMELOCKMIX); // and_b(timelock, heighlock) invalid + /* This is correctly detected as non-malleable but for the wrong reason. The type system assumes that branches 1 and 2 + can be spent together to create a non-malleble witness, but because of mixing of timelocks they cannot be spent together. + But since exactly one of the two after's can be satisfied, the witness involving the key cannot be malleated. + */ + Test("thresh(2,ltv:after(1000000000),altv:after(100),a:pk(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65))", "?", TESTMODE_VALID | TESTMODE_TIMELOCKMIX | TESTMODE_NONMAL); // thresh with k = 2 + // This is actually non-malleable in practice, but we cannot detect it in type system. See above rationale + Test("thresh(1,c:pk_k(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),altv:after(1000000000),altv:after(100))", "?", TESTMODE_VALID); // thresh with k = 1 + + + g_testdata.reset(); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/net_peer_eviction_tests.cpp b/src/test/net_peer_eviction_tests.cpp index 6ec3fb0c6b..d519a4442f 100644 --- a/src/test/net_peer_eviction_tests.cpp +++ b/src/test/net_peer_eviction_tests.cpp @@ -478,8 +478,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_network = NET_IPV6; } }, - /* protected_peer_ids */ {0, 4}, - /* unprotected_peer_ids */ {1, 2, 3}, + /*protected_peer_ids=*/{0, 4}, + /*unprotected_peer_ids=*/{1, 2, 3}, random_context)); // Combined test: expect having 1 CJDNS, 1 I2P, 1 localhost and 1 onion peer @@ -627,7 +627,7 @@ BOOST_AUTO_TEST_CASE(peer_eviction_test) number_of_nodes, [number_of_nodes](NodeEvictionCandidate& candidate) { candidate.m_last_block_time = std::chrono::seconds{number_of_nodes - candidate.id}; if (candidate.id <= 7) { - candidate.fRelayTxes = false; + candidate.m_relay_txs = false; candidate.fRelevantServices = true; } }, @@ -646,7 +646,7 @@ BOOST_AUTO_TEST_CASE(peer_eviction_test) number_of_nodes, [number_of_nodes](NodeEvictionCandidate& candidate) { candidate.m_last_block_time = std::chrono::seconds{number_of_nodes - candidate.id}; if (candidate.id <= 7) { - candidate.fRelayTxes = false; + candidate.m_relay_txs = false; candidate.fRelevantServices = true; } }, diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index fcb1a80765..e7c01bd6d0 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -732,26 +732,31 @@ BOOST_AUTO_TEST_CASE(LimitedAndReachable_Network) BOOST_CHECK(IsReachable(NET_IPV6)); BOOST_CHECK(IsReachable(NET_ONION)); BOOST_CHECK(IsReachable(NET_I2P)); + BOOST_CHECK(IsReachable(NET_CJDNS)); SetReachable(NET_IPV4, false); SetReachable(NET_IPV6, false); SetReachable(NET_ONION, false); SetReachable(NET_I2P, false); + SetReachable(NET_CJDNS, false); BOOST_CHECK(!IsReachable(NET_IPV4)); BOOST_CHECK(!IsReachable(NET_IPV6)); BOOST_CHECK(!IsReachable(NET_ONION)); BOOST_CHECK(!IsReachable(NET_I2P)); + BOOST_CHECK(!IsReachable(NET_CJDNS)); SetReachable(NET_IPV4, true); SetReachable(NET_IPV6, true); SetReachable(NET_ONION, true); SetReachable(NET_I2P, true); + SetReachable(NET_CJDNS, true); BOOST_CHECK(IsReachable(NET_IPV4)); BOOST_CHECK(IsReachable(NET_IPV6)); BOOST_CHECK(IsReachable(NET_ONION)); BOOST_CHECK(IsReachable(NET_I2P)); + BOOST_CHECK(IsReachable(NET_CJDNS)); } BOOST_AUTO_TEST_CASE(LimitedAndReachable_NetworkCaseUnroutableAndInternal) diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 8e6e911ec2..224dc88d0f 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -5,6 +5,7 @@ #include <net_permissions.h> #include <netaddress.h> #include <netbase.h> +#include <netgroup.h> #include <protocol.h> #include <serialize.h> #include <streams.h> @@ -315,22 +316,22 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_AUTO_TEST_CASE(netbase_getgroup) { - std::vector<bool> asmap; // use /16 - BOOST_CHECK(ResolveIP("127.0.0.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // Local -> !Routable() - BOOST_CHECK(ResolveIP("257.0.0.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // !Valid -> !Routable() - BOOST_CHECK(ResolveIP("10.0.0.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // RFC1918 -> !Routable() - BOOST_CHECK(ResolveIP("169.254.1.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // RFC3927 -> !Routable() - BOOST_CHECK(ResolveIP("1.2.3.4").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // IPv4 - BOOST_CHECK(ResolveIP("::FFFF:0:102:304").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6145 - BOOST_CHECK(ResolveIP("64:FF9B::102:304").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6052 - BOOST_CHECK(ResolveIP("2002:102:304:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC3964 - BOOST_CHECK(ResolveIP("2001:0:9999:9999:9999:9999:FEFD:FCFB").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC4380 - BOOST_CHECK(ResolveIP("2001:470:abcd:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 4, 112, 175})); //he.net - BOOST_CHECK(ResolveIP("2001:2001:9999:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 32, 1})); //IPv6 + NetGroupManager netgroupman{std::vector<bool>()}; // use /16 + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("127.0.0.1")) == std::vector<unsigned char>({0})); // Local -> !Routable() + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("257.0.0.1")) == std::vector<unsigned char>({0})); // !Valid -> !Routable() + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("10.0.0.1")) == std::vector<unsigned char>({0})); // RFC1918 -> !Routable() + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("169.254.1.1")) == std::vector<unsigned char>({0})); // RFC3927 -> !Routable() + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("1.2.3.4")) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // IPv4 + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("::FFFF:0:102:304")) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6145 + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("64:FF9B::102:304")) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6052 + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("2002:102:304:9999:9999:9999:9999:9999")) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC3964 + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("2001:0:9999:9999:9999:9999:FEFD:FCFB")) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC4380 + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("2001:470:abcd:9999:9999:9999:9999:9999")) == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 4, 112, 175})); //he.net + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("2001:2001:9999:9999:9999:9999:9999:9999")) == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 32, 1})); //IPv6 // baz.net sha256 hash: 12929400eb4607c4ac075f087167e75286b179c693eb059a01774b864e8fe505 std::vector<unsigned char> internal_group = {NET_INTERNAL, 0x12, 0x92, 0x94, 0x00, 0xeb, 0x46, 0x07, 0xc4, 0xac, 0x07}; - BOOST_CHECK(CreateInternal("baz.net").GetGroup(asmap) == internal_group); + BOOST_CHECK(netgroupman.GetGroup(CreateInternal("baz.net")) == internal_group); } BOOST_AUTO_TEST_CASE(netbase_parsenetwork) diff --git a/src/test/orphanage_tests.cpp b/src/test/orphanage_tests.cpp new file mode 100644 index 0000000000..842daa8bd4 --- /dev/null +++ b/src/test/orphanage_tests.cpp @@ -0,0 +1,137 @@ +// Copyright (c) 2011-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <arith_uint256.h> +#include <pubkey.h> +#include <script/sign.h> +#include <script/signingprovider.h> +#include <script/standard.h> +#include <test/util/setup_common.h> +#include <txorphanage.h> + +#include <array> +#include <cstdint> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(orphanage_tests, TestingSetup) + +class TxOrphanageTest : public TxOrphanage +{ +public: + inline size_t CountOrphans() const EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans) + { + return m_orphans.size(); + } + + CTransactionRef RandomOrphan() EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans) + { + std::map<uint256, OrphanTx>::iterator it; + it = m_orphans.lower_bound(InsecureRand256()); + if (it == m_orphans.end()) + it = m_orphans.begin(); + return it->second.tx; + } +}; + +static void MakeNewKeyWithFastRandomContext(CKey& key) +{ + std::vector<unsigned char> keydata; + keydata = g_insecure_rand_ctx.randbytes(32); + key.Set(keydata.data(), keydata.data() + keydata.size(), /*fCompressedIn=*/true); + assert(key.IsValid()); +} + +BOOST_AUTO_TEST_CASE(DoS_mapOrphans) +{ + // This test had non-deterministic coverage due to + // randomly selected seeds. + // This seed is chosen so that all branches of the function + // ecdsa_signature_parse_der_lax are executed during this test. + // Specifically branches that run only when an ECDSA + // signature's R and S values have leading zeros. + g_insecure_rand_ctx = FastRandomContext{uint256{33}}; + + TxOrphanageTest orphanage; + CKey key; + MakeNewKeyWithFastRandomContext(key); + FillableSigningProvider keystore; + BOOST_CHECK(keystore.AddKey(key)); + + LOCK(g_cs_orphans); + + // 50 orphan transactions: + for (int i = 0; i < 50; i++) + { + CMutableTransaction tx; + tx.vin.resize(1); + tx.vin[0].prevout.n = 0; + tx.vin[0].prevout.hash = InsecureRand256(); + tx.vin[0].scriptSig << OP_1; + tx.vout.resize(1); + tx.vout[0].nValue = 1*CENT; + tx.vout[0].scriptPubKey = GetScriptForDestination(PKHash(key.GetPubKey())); + + orphanage.AddTx(MakeTransactionRef(tx), i); + } + + // ... and 50 that depend on other orphans: + for (int i = 0; i < 50; i++) + { + CTransactionRef txPrev = orphanage.RandomOrphan(); + + CMutableTransaction tx; + tx.vin.resize(1); + tx.vin[0].prevout.n = 0; + tx.vin[0].prevout.hash = txPrev->GetHash(); + tx.vout.resize(1); + tx.vout[0].nValue = 1*CENT; + tx.vout[0].scriptPubKey = GetScriptForDestination(PKHash(key.GetPubKey())); + BOOST_CHECK(SignSignature(keystore, *txPrev, tx, 0, SIGHASH_ALL)); + + orphanage.AddTx(MakeTransactionRef(tx), i); + } + + // This really-big orphan should be ignored: + for (int i = 0; i < 10; i++) + { + CTransactionRef txPrev = orphanage.RandomOrphan(); + + CMutableTransaction tx; + tx.vout.resize(1); + tx.vout[0].nValue = 1*CENT; + tx.vout[0].scriptPubKey = GetScriptForDestination(PKHash(key.GetPubKey())); + tx.vin.resize(2777); + for (unsigned int j = 0; j < tx.vin.size(); j++) + { + tx.vin[j].prevout.n = j; + tx.vin[j].prevout.hash = txPrev->GetHash(); + } + BOOST_CHECK(SignSignature(keystore, *txPrev, tx, 0, SIGHASH_ALL)); + // Re-use same signature for other inputs + // (they don't have to be valid for this test) + for (unsigned int j = 1; j < tx.vin.size(); j++) + tx.vin[j].scriptSig = tx.vin[0].scriptSig; + + BOOST_CHECK(!orphanage.AddTx(MakeTransactionRef(tx), i)); + } + + // Test EraseOrphansFor: + for (NodeId i = 0; i < 3; i++) + { + size_t sizeBefore = orphanage.CountOrphans(); + orphanage.EraseForPeer(i); + BOOST_CHECK(orphanage.CountOrphans() < sizeBefore); + } + + // Test LimitOrphanTxSize() function: + orphanage.LimitOrphans(40); + BOOST_CHECK(orphanage.CountOrphans() <= 40); + orphanage.LimitOrphans(10); + BOOST_CHECK(orphanage.CountOrphans() <= 10); + orphanage.LimitOrphans(0); + BOOST_CHECK(orphanage.CountOrphans() == 0); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/pmt_tests.cpp b/src/test/pmt_tests.cpp index a9d661438c..3cbbec92d6 100644 --- a/src/test/pmt_tests.cpp +++ b/src/test/pmt_tests.cpp @@ -2,7 +2,6 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <arith_uint256.h> #include <consensus/merkle.h> #include <merkleblock.h> #include <serialize.h> @@ -107,13 +106,13 @@ BOOST_AUTO_TEST_CASE(pmt_test1) BOOST_AUTO_TEST_CASE(pmt_malleability) { - std::vector<uint256> vTxid = { - ArithToUint256(1), ArithToUint256(2), - ArithToUint256(3), ArithToUint256(4), - ArithToUint256(5), ArithToUint256(6), - ArithToUint256(7), ArithToUint256(8), - ArithToUint256(9), ArithToUint256(10), - ArithToUint256(9), ArithToUint256(10), + std::vector<uint256> vTxid{ + uint256{1}, uint256{2}, + uint256{3}, uint256{4}, + uint256{5}, uint256{6}, + uint256{7}, uint256{8}, + uint256{9}, uint256{10}, + uint256{9}, uint256{10}, }; std::vector<bool> vMatch = {false, false, false, false, false, false, false, false, false, true, true, false}; diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index 89814748fe..3977a3d548 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -165,7 +165,8 @@ public: test(); } - void swap() { + void swap() noexcept + { real_vector.swap(real_vector_alt); pre_vector.swap(pre_vector_alt); test(); diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index 978a7bee4d..a3daefa599 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -31,6 +31,16 @@ BOOST_AUTO_TEST_CASE(fastrandom_tests) BOOST_CHECK_EQUAL(GetRandMicros(std::chrono::hours{1}).count(), 2917185654); BOOST_CHECK_EQUAL(GetRandMillis(std::chrono::hours{1}).count(), 2144374); } + { + constexpr SteadySeconds time_point{1s}; + FastRandomContext ctx{true}; + BOOST_CHECK_EQUAL(7, ctx.rand_uniform_delay(time_point, 9s).time_since_epoch().count()); + BOOST_CHECK_EQUAL(-6, ctx.rand_uniform_delay(time_point, -9s).time_since_epoch().count()); + BOOST_CHECK_EQUAL(1, ctx.rand_uniform_delay(time_point, 0s).time_since_epoch().count()); + BOOST_CHECK_EQUAL(1467825113502396065, ctx.rand_uniform_delay(time_point, 9223372036854775807s).time_since_epoch().count()); + BOOST_CHECK_EQUAL(-970181367944767837, ctx.rand_uniform_delay(time_point, -9223372036854775807s).time_since_epoch().count()); + BOOST_CHECK_EQUAL(24761, ctx.rand_uniform_delay(time_point, 9h).time_since_epoch().count()); + } BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); BOOST_CHECK_EQUAL(ctx1.rand64(), ctx2.rand64()); diff --git a/src/test/rest_tests.cpp b/src/test/rest_tests.cpp new file mode 100644 index 0000000000..20dfe4b41a --- /dev/null +++ b/src/test/rest_tests.cpp @@ -0,0 +1,48 @@ +// Copyright (c) 2012-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <rest.h> +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +#include <string> + +BOOST_FIXTURE_TEST_SUITE(rest_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(test_query_string) +{ + std::string param; + RESTResponseFormat rf; + // No query string + rf = ParseDataFormat(param, "/rest/endpoint/someresource.json"); + BOOST_CHECK_EQUAL(param, "/rest/endpoint/someresource"); + BOOST_CHECK_EQUAL(rf, RESTResponseFormat::JSON); + + // Query string with single parameter + rf = ParseDataFormat(param, "/rest/endpoint/someresource.bin?p1=v1"); + BOOST_CHECK_EQUAL(param, "/rest/endpoint/someresource"); + BOOST_CHECK_EQUAL(rf, RESTResponseFormat::BINARY); + + // Query string with multiple parameters + rf = ParseDataFormat(param, "/rest/endpoint/someresource.hex?p1=v1&p2=v2"); + BOOST_CHECK_EQUAL(param, "/rest/endpoint/someresource"); + BOOST_CHECK_EQUAL(rf, RESTResponseFormat::HEX); + + // Incorrectly formed query string will not be handled + rf = ParseDataFormat(param, "/rest/endpoint/someresource.json&p1=v1"); + BOOST_CHECK_EQUAL(param, "/rest/endpoint/someresource.json&p1=v1"); + BOOST_CHECK_EQUAL(rf, RESTResponseFormat::UNDEF); + + // Omitted data format with query string should return UNDEF and hide query string + rf = ParseDataFormat(param, "/rest/endpoint/someresource?p1=v1"); + BOOST_CHECK_EQUAL(param, "/rest/endpoint/someresource"); + BOOST_CHECK_EQUAL(rf, RESTResponseFormat::UNDEF); + + // Data format specified after query string + rf = ParseDataFormat(param, "/rest/endpoint/someresource?p1=v1.json"); + BOOST_CHECK_EQUAL(param, "/rest/endpoint/someresource"); + BOOST_CHECK_EQUAL(rf, RESTResponseFormat::UNDEF); +} +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index 50b5078110..7cec287e8f 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -2,25 +2,21 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <rpc/client.h> -#include <rpc/server.h> -#include <rpc/util.h> - #include <core_io.h> #include <interfaces/chain.h> #include <node/context.h> +#include <rpc/blockchain.h> +#include <rpc/client.h> +#include <rpc/server.h> +#include <rpc/util.h> #include <test/util/setup_common.h> +#include <univalue.h> #include <util/time.h> #include <any> -#include <boost/algorithm/string.hpp> #include <boost/test/unit_test.hpp> -#include <univalue.h> - -#include <rpc/blockchain.h> - class RPCTestingSetup : public TestingSetup { public: @@ -29,8 +25,7 @@ public: UniValue RPCTestingSetup::CallRPC(std::string args) { - std::vector<std::string> vArgs; - boost::split(vArgs, args, boost::is_any_of(" \t")); + std::vector<std::string> vArgs{SplitString(args, ' ')}; std::string strMethod = vArgs[0]; vArgs.erase(vArgs.begin()); JSONRPCRequest request; diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp index 4195d413fc..7b5dda8114 100644 --- a/src/test/scheduler_tests.cpp +++ b/src/test/scheduler_tests.cpp @@ -15,13 +15,13 @@ BOOST_AUTO_TEST_SUITE(scheduler_tests) -static void microTask(CScheduler& s, std::mutex& mutex, int& counter, int delta, std::chrono::system_clock::time_point rescheduleTime) +static void microTask(CScheduler& s, std::mutex& mutex, int& counter, int delta, std::chrono::steady_clock::time_point rescheduleTime) { { std::lock_guard<std::mutex> lock(mutex); counter += delta; } - std::chrono::system_clock::time_point noTime = std::chrono::system_clock::time_point::min(); + auto noTime = std::chrono::steady_clock::time_point::min(); if (rescheduleTime != noTime) { CScheduler::Function f = std::bind(µTask, std::ref(s), std::ref(mutex), std::ref(counter), -delta + 1, noTime); s.schedule(f, rescheduleTime); @@ -49,15 +49,15 @@ BOOST_AUTO_TEST_CASE(manythreads) auto randomMsec = [](FastRandomContext& rc) -> int { return -11 + (int)rc.randrange(1012); }; // [-11, 1000] auto randomDelta = [](FastRandomContext& rc) -> int { return -1000 + (int)rc.randrange(2001); }; // [-1000, 1000] - std::chrono::system_clock::time_point start = std::chrono::system_clock::now(); - std::chrono::system_clock::time_point now = start; - std::chrono::system_clock::time_point first, last; + auto start = std::chrono::steady_clock::now(); + auto now = start; + std::chrono::steady_clock::time_point first, last; size_t nTasks = microTasks.getQueueInfo(first, last); BOOST_CHECK(nTasks == 0); for (int i = 0; i < 100; ++i) { - std::chrono::system_clock::time_point t = now + std::chrono::microseconds(randomMsec(rng)); - std::chrono::system_clock::time_point tReschedule = now + std::chrono::microseconds(500 + randomMsec(rng)); + auto t = now + std::chrono::microseconds(randomMsec(rng)); + auto tReschedule = now + std::chrono::microseconds(500 + randomMsec(rng)); int whichCounter = zeroToNine(rng); CScheduler::Function f = std::bind(µTask, std::ref(microTasks), std::ref(counterMutex[whichCounter]), std::ref(counter[whichCounter]), @@ -75,14 +75,14 @@ BOOST_AUTO_TEST_CASE(manythreads) microThreads.emplace_back(std::bind(&CScheduler::serviceQueue, µTasks)); UninterruptibleSleep(std::chrono::microseconds{600}); - now = std::chrono::system_clock::now(); + now = std::chrono::steady_clock::now(); // More threads and more tasks: for (int i = 0; i < 5; i++) microThreads.emplace_back(std::bind(&CScheduler::serviceQueue, µTasks)); for (int i = 0; i < 100; i++) { - std::chrono::system_clock::time_point t = now + std::chrono::microseconds(randomMsec(rng)); - std::chrono::system_clock::time_point tReschedule = now + std::chrono::microseconds(500 + randomMsec(rng)); + auto t = now + std::chrono::microseconds(randomMsec(rng)); + auto tReschedule = now + std::chrono::microseconds(500 + randomMsec(rng)); int whichCounter = zeroToNine(rng); CScheduler::Function f = std::bind(µTask, std::ref(microTasks), std::ref(counterMutex[whichCounter]), std::ref(counter[whichCounter]), @@ -111,8 +111,8 @@ BOOST_AUTO_TEST_CASE(wait_until_past) Mutex mtx; WAIT_LOCK(mtx, lock); - const auto no_wait= [&](const std::chrono::seconds& d) { - return condvar.wait_until(lock, std::chrono::system_clock::now() - d); + const auto no_wait = [&](const std::chrono::seconds& d) { + return condvar.wait_until(lock, std::chrono::steady_clock::now() - d); }; BOOST_CHECK(std::cv_status::timeout == no_wait(std::chrono::seconds{1})); @@ -128,8 +128,8 @@ BOOST_AUTO_TEST_CASE(singlethreadedscheduler_ordered) CScheduler scheduler; // each queue should be well ordered with respect to itself but not other queues - SingleThreadedSchedulerClient queue1(&scheduler); - SingleThreadedSchedulerClient queue2(&scheduler); + SingleThreadedSchedulerClient queue1(scheduler); + SingleThreadedSchedulerClient queue2(scheduler); // create more threads than queues // if the queues only permit execution of one task at once then @@ -137,7 +137,7 @@ BOOST_AUTO_TEST_CASE(singlethreadedscheduler_ordered) // if they don't we'll get out of order behaviour std::vector<std::thread> threads; for (int i = 0; i < 5; ++i) { - threads.emplace_back(std::bind(&CScheduler::serviceQueue, &scheduler)); + threads.emplace_back([&] { scheduler.serviceQueue(); }); } // these are not atomic, if SinglethreadedSchedulerClient prevents @@ -183,7 +183,7 @@ BOOST_AUTO_TEST_CASE(mockforward) scheduler.scheduleFromNow(dummy, std::chrono::minutes{8}); // check taskQueue - std::chrono::system_clock::time_point first, last; + std::chrono::steady_clock::time_point first, last; size_t num_tasks = scheduler.getQueueInfo(first, last); BOOST_CHECK_EQUAL(num_tasks, 3ul); @@ -204,7 +204,7 @@ BOOST_AUTO_TEST_CASE(mockforward) BOOST_CHECK_EQUAL(counter, 2); // check that the time of the remaining job has been updated - std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); + auto now = std::chrono::steady_clock::now(); int delta = std::chrono::duration_cast<std::chrono::seconds>(first - now).count(); // should be between 2 & 3 minutes from now BOOST_CHECK(delta > 2*60 && delta < 3*60); diff --git a/src/test/script_segwit_tests.cpp b/src/test/script_segwit_tests.cpp new file mode 100644 index 0000000000..2bad59805f --- /dev/null +++ b/src/test/script_segwit_tests.cpp @@ -0,0 +1,164 @@ +// Copyright (c) 2012-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <script/script.h> +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(script_segwit_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(IsPayToWitnessScriptHash_Valid) +{ + uint256 dummy; + CScript p2wsh; + p2wsh << OP_0 << ToByteVector(dummy); + BOOST_CHECK(p2wsh.IsPayToWitnessScriptHash()); + + std::vector<unsigned char> bytes = {OP_0, 32}; + bytes.insert(bytes.end(), 32, 0); + BOOST_CHECK(CScript(bytes.begin(), bytes.end()).IsPayToWitnessScriptHash()); +} + +BOOST_AUTO_TEST_CASE(IsPayToWitnessScriptHash_Invalid_NotOp0) +{ + uint256 dummy; + CScript notp2wsh; + notp2wsh << OP_1 << ToByteVector(dummy); + BOOST_CHECK(!notp2wsh.IsPayToWitnessScriptHash()); +} + +BOOST_AUTO_TEST_CASE(IsPayToWitnessScriptHash_Invalid_Size) +{ + uint160 dummy; + CScript notp2wsh; + notp2wsh << OP_0 << ToByteVector(dummy); + BOOST_CHECK(!notp2wsh.IsPayToWitnessScriptHash()); +} + +BOOST_AUTO_TEST_CASE(IsPayToWitnessScriptHash_Invalid_Nop) +{ + uint256 dummy; + CScript notp2wsh; + notp2wsh << OP_0 << OP_NOP << ToByteVector(dummy); + BOOST_CHECK(!notp2wsh.IsPayToWitnessScriptHash()); +} + +BOOST_AUTO_TEST_CASE(IsPayToWitnessScriptHash_Invalid_EmptyScript) +{ + CScript notp2wsh; + BOOST_CHECK(!notp2wsh.IsPayToWitnessScriptHash()); +} + +BOOST_AUTO_TEST_CASE(IsPayToWitnessScriptHash_Invalid_Pushdata) +{ + // A script is not P2WSH if OP_PUSHDATA is used to push the hash. + std::vector<unsigned char> bytes = {OP_0, OP_PUSHDATA1, 32}; + bytes.insert(bytes.end(), 32, 0); + BOOST_CHECK(!CScript(bytes.begin(), bytes.end()).IsPayToWitnessScriptHash()); + + bytes = {OP_0, OP_PUSHDATA2, 32, 0}; + bytes.insert(bytes.end(), 32, 0); + BOOST_CHECK(!CScript(bytes.begin(), bytes.end()).IsPayToWitnessScriptHash()); + + bytes = {OP_0, OP_PUSHDATA4, 32, 0, 0, 0}; + bytes.insert(bytes.end(), 32, 0); + BOOST_CHECK(!CScript(bytes.begin(), bytes.end()).IsPayToWitnessScriptHash()); +} + +namespace { + +bool IsExpectedWitnessProgram(const CScript& script, const int expectedVersion, const std::vector<unsigned char>& expectedProgram) +{ + int actualVersion; + std::vector<unsigned char> actualProgram; + if (!script.IsWitnessProgram(actualVersion, actualProgram)) { + return false; + } + BOOST_CHECK_EQUAL(actualVersion, expectedVersion); + BOOST_CHECK(actualProgram == expectedProgram); + return true; +} + +bool IsNoWitnessProgram(const CScript& script) +{ + int dummyVersion; + std::vector<unsigned char> dummyProgram; + return !script.IsWitnessProgram(dummyVersion, dummyProgram); +} + +} // anonymous namespace + +BOOST_AUTO_TEST_CASE(IsWitnessProgram_Valid) +{ + // Witness programs have a minimum data push of 2 bytes. + std::vector<unsigned char> program = {42, 18}; + CScript wit; + wit << OP_0 << program; + BOOST_CHECK(IsExpectedWitnessProgram(wit, 0, program)); + + wit.clear(); + // Witness programs have a maximum data push of 40 bytes. + program.resize(40); + wit << OP_16 << program; + BOOST_CHECK(IsExpectedWitnessProgram(wit, 16, program)); + + program.resize(32); + std::vector<unsigned char> bytes = {OP_5, static_cast<unsigned char>(program.size())}; + bytes.insert(bytes.end(), program.begin(), program.end()); + BOOST_CHECK(IsExpectedWitnessProgram(CScript(bytes.begin(), bytes.end()), 5, program)); +} + +BOOST_AUTO_TEST_CASE(IsWitnessProgram_Invalid_Version) +{ + std::vector<unsigned char> program(10); + CScript nowit; + nowit << OP_1NEGATE << program; + BOOST_CHECK(IsNoWitnessProgram(nowit)); +} + +BOOST_AUTO_TEST_CASE(IsWitnessProgram_Invalid_Size) +{ + std::vector<unsigned char> program(1); + CScript nowit; + nowit << OP_0 << program; + BOOST_CHECK(IsNoWitnessProgram(nowit)); + + nowit.clear(); + program.resize(41); + nowit << OP_0 << program; + BOOST_CHECK(IsNoWitnessProgram(nowit)); +} + +BOOST_AUTO_TEST_CASE(IsWitnessProgram_Invalid_Nop) +{ + std::vector<unsigned char> program(10); + CScript nowit; + nowit << OP_0 << OP_NOP << program; + BOOST_CHECK(IsNoWitnessProgram(nowit)); +} + +BOOST_AUTO_TEST_CASE(IsWitnessProgram_Invalid_EmptyScript) +{ + CScript nowit; + BOOST_CHECK(IsNoWitnessProgram(nowit)); +} + +BOOST_AUTO_TEST_CASE(IsWitnessProgram_Invalid_Pushdata) +{ + // A script is no witness program if OP_PUSHDATA is used to push the hash. + std::vector<unsigned char> bytes = {OP_0, OP_PUSHDATA1, 32}; + bytes.insert(bytes.end(), 32, 0); + BOOST_CHECK(IsNoWitnessProgram(CScript(bytes.begin(), bytes.end()))); + + bytes = {OP_0, OP_PUSHDATA2, 32, 0}; + bytes.insert(bytes.end(), 32, 0); + BOOST_CHECK(IsNoWitnessProgram(CScript(bytes.begin(), bytes.end()))); + + bytes = {OP_0, OP_PUSHDATA4, 32, 0, 0, 0}; + bytes.insert(bytes.end(), 32, 0); + BOOST_CHECK(IsNoWitnessProgram(CScript(bytes.begin(), bytes.end()))); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index c453f22594..19e32d0c36 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -1162,7 +1162,7 @@ SignatureData CombineSignatures(const CTxOut& txout, const CMutableTransaction& SignatureData data; data.MergeSignatureData(scriptSig1); data.MergeSignatureData(scriptSig2); - ProduceSignature(DUMMY_SIGNING_PROVIDER, MutableTransactionSignatureCreator(&tx, 0, txout.nValue, SIGHASH_DEFAULT), txout.scriptPubKey, data); + ProduceSignature(DUMMY_SIGNING_PROVIDER, MutableTransactionSignatureCreator(tx, 0, txout.nValue, SIGHASH_DEFAULT), txout.scriptPubKey, data); return data; } @@ -1796,7 +1796,7 @@ BOOST_AUTO_TEST_CASE(bip341_keypath_test_vectors) // Sign and verify signature. FlatSigningProvider provider; provider.keys[key.GetPubKey().GetID()] = key; - MutableTransactionSignatureCreator creator(&tx, txinpos, utxos[txinpos].nValue, &txdata, hashtype); + MutableTransactionSignatureCreator creator(tx, txinpos, utxos[txinpos].nValue, &txdata, hashtype); std::vector<unsigned char> signature; BOOST_CHECK(creator.CreateSchnorrSig(provider, signature, pubkey, nullptr, &merkle_root, SigVersion::TAPROOT)); BOOST_CHECK_EQUAL(HexStr(signature), input["expected"]["witness"][0].get_str()); diff --git a/src/test/system_tests.cpp b/src/test/system_tests.cpp index 9c6950f11f..3f5353b5a2 100644 --- a/src/test/system_tests.cpp +++ b/src/test/system_tests.cpp @@ -12,7 +12,16 @@ // For details see https://github.com/bitcoin/bitcoin/pull/22348. #define __kernel_entry #endif +#if defined(__GNUC__) +// Boost 1.78 requires the following workaround. +// See: https://github.com/boostorg/process/issues/235 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wnarrowing" +#endif #include <boost/process.hpp> +#if defined(__GNUC__) +#pragma GCC diagnostic pop +#endif #endif // ENABLE_EXTERNAL_SIGNER #include <boost/test/unit_test.hpp> diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 4fb7f9c405..c3a2a66027 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -31,8 +31,6 @@ #include <map> #include <string> -#include <boost/algorithm/string/classification.hpp> -#include <boost/algorithm/string/split.hpp> #include <boost/test/unit_test.hpp> #include <univalue.h> @@ -70,8 +68,7 @@ unsigned int ParseScriptFlags(std::string strFlags) { if (strFlags.empty() || strFlags == "NONE") return 0; unsigned int flags = 0; - std::vector<std::string> words; - boost::algorithm::split(words, strFlags, boost::algorithm::is_any_of(",")); + std::vector<std::string> words = SplitString(strFlags, ','); for (const std::string& word : words) { @@ -562,7 +559,7 @@ SignatureData CombineSignatures(const CMutableTransaction& input1, const CMutabl SignatureData sigdata; sigdata = DataFromTransaction(input1, 0, tx->vout[0]); sigdata.MergeSignatureData(DataFromTransaction(input2, 0, tx->vout[0])); - ProduceSignature(DUMMY_SIGNING_PROVIDER, MutableTransactionSignatureCreator(&input1, 0, tx->vout[0].nValue, SIGHASH_ALL), tx->vout[0].scriptPubKey, sigdata); + ProduceSignature(DUMMY_SIGNING_PROVIDER, MutableTransactionSignatureCreator(input1, 0, tx->vout[0].nValue, SIGHASH_ALL), tx->vout[0].scriptPubKey, sigdata); return sigdata; } diff --git a/src/test/txpackage_tests.cpp b/src/test/txpackage_tests.cpp index 6013080dc6..079b753304 100644 --- a/src/test/txpackage_tests.cpp +++ b/src/test/txpackage_tests.cpp @@ -73,19 +73,19 @@ BOOST_FIXTURE_TEST_CASE(package_validation_tests, TestChain100Setup) CKey parent_key; parent_key.MakeNewKey(true); CScript parent_locking_script = GetScriptForDestination(PKHash(parent_key.GetPubKey())); - auto mtx_parent = CreateValidMempoolTransaction(/*input_transaction=*/ m_coinbase_txns[0], /*input_vout=*/0, - /*input_height=*/ 0, /*input_signing_key=*/coinbaseKey, - /*output_destination=*/ parent_locking_script, - /*output_amount=*/ CAmount(49 * COIN), /*submit=*/false); + auto mtx_parent = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[0], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/parent_locking_script, + /*output_amount=*/CAmount(49 * COIN), /*submit=*/false); CTransactionRef tx_parent = MakeTransactionRef(mtx_parent); CKey child_key; child_key.MakeNewKey(true); CScript child_locking_script = GetScriptForDestination(PKHash(child_key.GetPubKey())); - auto mtx_child = CreateValidMempoolTransaction(/*input_transaction=*/ tx_parent, /*input_vout=*/0, - /*input_height=*/ 101, /*input_signing_key=*/parent_key, + auto mtx_child = CreateValidMempoolTransaction(/*input_transaction=*/tx_parent, /*input_vout=*/0, + /*input_height=*/101, /*input_signing_key=*/parent_key, /*output_destination=*/child_locking_script, - /*output_amount=*/ CAmount(48 * COIN), /*submit=*/false); + /*output_amount=*/CAmount(48 * COIN), /*submit=*/false); CTransactionRef tx_child = MakeTransactionRef(mtx_child); const auto result_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, {tx_parent, tx_child}, /*test_accept=*/true); BOOST_CHECK_MESSAGE(result_parent_child.m_state.IsValid(), @@ -98,7 +98,9 @@ BOOST_FIXTURE_TEST_CASE(package_validation_tests, TestChain100Setup) BOOST_CHECK(it_child != result_parent_child.m_tx_results.end()); BOOST_CHECK_MESSAGE(it_child->second.m_state.IsValid(), "Package validation unexpectedly failed: " << it_child->second.m_state.GetRejectReason()); - + BOOST_CHECK(result_parent_child.m_package_feerate.has_value()); + BOOST_CHECK(result_parent_child.m_package_feerate.value() == + CFeeRate(2 * COIN, GetVirtualTransactionSize(*tx_parent) + GetVirtualTransactionSize(*tx_child))); // A single, giant transaction submitted through ProcessNewPackage fails on single tx policy. CTransactionRef giant_ptx = create_placeholder_tx(999, 999); @@ -110,6 +112,7 @@ BOOST_FIXTURE_TEST_CASE(package_validation_tests, TestChain100Setup) auto it_giant_tx = result_single_large.m_tx_results.find(giant_ptx->GetWitnessHash()); BOOST_CHECK(it_giant_tx != result_single_large.m_tx_results.end()); BOOST_CHECK_EQUAL(it_giant_tx->second.m_state.GetRejectReason(), "tx-size"); + BOOST_CHECK(result_single_large.m_package_feerate == std::nullopt); // Check that mempool size hasn't changed. BOOST_CHECK_EQUAL(m_node.mempool->size(), initialPoolSize); @@ -128,11 +131,11 @@ BOOST_FIXTURE_TEST_CASE(noncontextual_package_tests, TestChain100Setup) // Parent and Child Package { auto mtx_parent = CreateValidMempoolTransaction(m_coinbase_txns[0], 0, 0, coinbaseKey, spk, - CAmount(49 * COIN), /* submit */ false); + CAmount(49 * COIN), /*submit=*/false); CTransactionRef tx_parent = MakeTransactionRef(mtx_parent); auto mtx_child = CreateValidMempoolTransaction(tx_parent, 0, 101, placeholder_key, spk2, - CAmount(48 * COIN), /* submit */ false); + CAmount(48 * COIN), /*submit=*/false); CTransactionRef tx_child = MakeTransactionRef(mtx_child); PackageValidationState state; @@ -218,26 +221,27 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) // Unrelated transactions are not allowed in package submission. Package package_unrelated; for (size_t i{0}; i < 10; ++i) { - auto mtx = CreateValidMempoolTransaction(/* input_transaction */ m_coinbase_txns[i + 25], /* vout */ 0, - /* input_height */ 0, /* input_signing_key */ coinbaseKey, - /* output_destination */ parent_locking_script, - /* output_amount */ CAmount(49 * COIN), /* submit */ false); + auto mtx = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[i + 25], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/parent_locking_script, + /*output_amount=*/CAmount(49 * COIN), /*submit=*/false); package_unrelated.emplace_back(MakeTransactionRef(mtx)); } auto result_unrelated_submit = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_unrelated, /* test_accept */ false); + package_unrelated, /*test_accept=*/false); BOOST_CHECK(result_unrelated_submit.m_state.IsInvalid()); BOOST_CHECK_EQUAL(result_unrelated_submit.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(result_unrelated_submit.m_state.GetRejectReason(), "package-not-child-with-parents"); BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + BOOST_CHECK(result_unrelated_submit.m_package_feerate == std::nullopt); // Parent and Child (and Grandchild) Package Package package_parent_child; Package package_3gen; - auto mtx_parent = CreateValidMempoolTransaction(/* input_transaction */ m_coinbase_txns[0], /* vout */ 0, - /* input_height */ 0, /* input_signing_key */ coinbaseKey, - /* output_destination */ parent_locking_script, - /* output_amount */ CAmount(49 * COIN), /* submit */ false); + auto mtx_parent = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[0], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/parent_locking_script, + /*output_amount=*/CAmount(49 * COIN), /*submit=*/false); CTransactionRef tx_parent = MakeTransactionRef(mtx_parent); package_parent_child.push_back(tx_parent); package_3gen.push_back(tx_parent); @@ -245,10 +249,10 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) CKey child_key; child_key.MakeNewKey(true); CScript child_locking_script = GetScriptForDestination(PKHash(child_key.GetPubKey())); - auto mtx_child = CreateValidMempoolTransaction(/* input_transaction */ tx_parent, /* vout */ 0, - /* input_height */ 101, /* input_signing_key */ parent_key, - /* output_destination */ child_locking_script, - /* output_amount */ CAmount(48 * COIN), /* submit */ false); + auto mtx_child = CreateValidMempoolTransaction(/*input_transaction=*/tx_parent, /*input_vout=*/0, + /*input_height=*/101, /*input_signing_key=*/parent_key, + /*output_destination=*/child_locking_script, + /*output_amount=*/CAmount(48 * COIN), /*submit=*/false); CTransactionRef tx_child = MakeTransactionRef(mtx_child); package_parent_child.push_back(tx_child); package_3gen.push_back(tx_child); @@ -256,21 +260,22 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) CKey grandchild_key; grandchild_key.MakeNewKey(true); CScript grandchild_locking_script = GetScriptForDestination(PKHash(grandchild_key.GetPubKey())); - auto mtx_grandchild = CreateValidMempoolTransaction(/* input_transaction */ tx_child, /* vout */ 0, - /* input_height */ 101, /* input_signing_key */ child_key, - /* output_destination */ grandchild_locking_script, - /* output_amount */ CAmount(47 * COIN), /* submit */ false); + auto mtx_grandchild = CreateValidMempoolTransaction(/*input_transaction=*/tx_child, /*input_vout=*/0, + /*input_height=*/101, /*input_signing_key=*/child_key, + /*output_destination=*/grandchild_locking_script, + /*output_amount=*/CAmount(47 * COIN), /*submit=*/false); CTransactionRef tx_grandchild = MakeTransactionRef(mtx_grandchild); package_3gen.push_back(tx_grandchild); // 3 Generations is not allowed. { auto result_3gen_submit = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_3gen, /* test_accept */ false); + package_3gen, /*test_accept=*/false); BOOST_CHECK(result_3gen_submit.m_state.IsInvalid()); BOOST_CHECK_EQUAL(result_3gen_submit.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(result_3gen_submit.m_state.GetRejectReason(), "package-not-child-with-parents"); BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + BOOST_CHECK(result_3gen_submit.m_package_feerate == std::nullopt); } // Child with missing parent. @@ -280,18 +285,19 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) package_missing_parent.push_back(MakeTransactionRef(mtx_child)); { const auto result_missing_parent = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_missing_parent, /* test_accept */ false); + package_missing_parent, /*test_accept=*/false); BOOST_CHECK(result_missing_parent.m_state.IsInvalid()); BOOST_CHECK_EQUAL(result_missing_parent.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(result_missing_parent.m_state.GetRejectReason(), "package-not-child-with-unconfirmed-parents"); BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + BOOST_CHECK(result_missing_parent.m_package_feerate == std::nullopt); } // Submit package with parent + child. { const auto submit_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_parent_child, /* test_accept */ false); + package_parent_child, /*test_accept=*/false); expected_pool_size += 2; BOOST_CHECK_MESSAGE(submit_parent_child.m_state.IsValid(), "Package validation unexpectedly failed: " << submit_parent_child.m_state.GetRejectReason()); @@ -305,12 +311,16 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_parent->GetHash()))); BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_child->GetHash()))); + + // Since both transactions have high feerates, they each passed validation individually. + // Package validation was unnecessary, so there is no package feerate. + BOOST_CHECK(submit_parent_child.m_package_feerate == std::nullopt); } // Already-in-mempool transactions should be detected and de-duplicated. { const auto submit_deduped = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_parent_child, /* test_accept */ false); + package_parent_child, /*test_accept=*/false); BOOST_CHECK_MESSAGE(submit_deduped.m_state.IsValid(), "Package validation unexpectedly failed: " << submit_deduped.m_state.GetRejectReason()); auto it_parent_deduped = submit_deduped.m_tx_results.find(tx_parent->GetWitnessHash()); @@ -325,6 +335,8 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_parent->GetHash()))); BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_child->GetHash()))); + + BOOST_CHECK(submit_deduped.m_package_feerate == std::nullopt); } } @@ -340,10 +352,10 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // and the mempool entry's wtxid returned. CScript witnessScript = CScript() << OP_DROP << OP_TRUE; CScript scriptPubKey = GetScriptForDestination(WitnessV0ScriptHash(witnessScript)); - auto mtx_parent = CreateValidMempoolTransaction(/*input_transaction=*/ m_coinbase_txns[0], /*vout=*/ 0, - /*input_height=*/ 0, /*input_signing_key=*/ coinbaseKey, - /*output_destination=*/ scriptPubKey, - /*output_amount=*/ CAmount(49 * COIN), /*submit=*/ false); + auto mtx_parent = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[0], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/scriptPubKey, + /*output_amount=*/CAmount(49 * COIN), /*submit=*/false); CTransactionRef ptx_parent = MakeTransactionRef(mtx_parent); // Make two children with the same txid but different witnesses. @@ -384,7 +396,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // same-txid-different-witness. { const auto submit_witness1 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_parent, ptx_child1}, /*test_accept=*/ false); + {ptx_parent, ptx_child1}, /*test_accept=*/false); BOOST_CHECK_MESSAGE(submit_witness1.m_state.IsValid(), "Package validation unexpectedly failed: " << submit_witness1.m_state.GetRejectReason()); auto it_parent1 = submit_witness1.m_tx_results.find(ptx_parent->GetWitnessHash()); @@ -399,8 +411,12 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent->GetHash()))); BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_child1->GetHash()))); + // Child2 would have been validated individually. + BOOST_CHECK(submit_witness1.m_package_feerate == std::nullopt); + const auto submit_witness2 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_parent, ptx_child2}, /*test_accept=*/ false); + {ptx_parent, ptx_child2}, /*test_accept=*/false); + BOOST_CHECK(submit_witness2.m_package_feerate == std::nullopt); BOOST_CHECK_MESSAGE(submit_witness2.m_state.IsValid(), "Package validation unexpectedly failed: " << submit_witness2.m_state.GetRejectReason()); auto it_parent2_deduped = submit_witness2.m_tx_results.find(ptx_parent->GetWitnessHash()); @@ -417,13 +433,14 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // Deduplication should work when wtxid != txid. Submit package with the already-in-mempool // transactions again, which should not fail. const auto submit_segwit_dedup = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_parent, ptx_child1}, /*test_accept=*/ false); + {ptx_parent, ptx_child1}, /*test_accept=*/false); BOOST_CHECK_MESSAGE(submit_segwit_dedup.m_state.IsValid(), "Package validation unexpectedly failed: " << submit_segwit_dedup.m_state.GetRejectReason()); auto it_parent_dup = submit_segwit_dedup.m_tx_results.find(ptx_parent->GetWitnessHash()); auto it_child_dup = submit_segwit_dedup.m_tx_results.find(ptx_child1->GetWitnessHash()); BOOST_CHECK(it_parent_dup->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); BOOST_CHECK(it_child_dup->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + BOOST_CHECK(submit_witness2.m_package_feerate == std::nullopt); } // Try submitting Package1{child2, grandchild} where child2 is same-txid-different-witness as @@ -436,16 +453,16 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) CKey grandchild_key; grandchild_key.MakeNewKey(true); CScript grandchild_locking_script = GetScriptForDestination(WitnessV0KeyHash(grandchild_key.GetPubKey())); - auto mtx_grandchild = CreateValidMempoolTransaction(/*input_transaction=*/ ptx_child2, /* vout=*/ 0, - /*input_height=*/ 0, /*input_signing_key=*/ child_key, - /*output_destination=*/ grandchild_locking_script, - /*output_amount=*/ CAmount(47 * COIN), /*submit=*/ false); + auto mtx_grandchild = CreateValidMempoolTransaction(/*input_transaction=*/ptx_child2, /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/child_key, + /*output_destination=*/grandchild_locking_script, + /*output_amount=*/CAmount(47 * COIN), /*submit=*/false); CTransactionRef ptx_grandchild = MakeTransactionRef(mtx_grandchild); // We already submitted child1 above. { const auto submit_spend_ignored = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_child2, ptx_grandchild}, /*test_accept=*/ false); + {ptx_child2, ptx_grandchild}, /*test_accept=*/false); BOOST_CHECK_MESSAGE(submit_spend_ignored.m_state.IsValid(), "Package validation unexpectedly failed: " << submit_spend_ignored.m_state.GetRejectReason()); auto it_child2_ignored = submit_spend_ignored.m_tx_results.find(ptx_child2->GetWitnessHash()); @@ -458,6 +475,9 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_child2->GetHash()))); BOOST_CHECK(!m_node.mempool->exists(GenTxid::Wtxid(ptx_child2->GetWitnessHash()))); BOOST_CHECK(m_node.mempool->exists(GenTxid::Wtxid(ptx_grandchild->GetWitnessHash()))); + + // Since child2 is ignored, grandchild would be validated individually. + BOOST_CHECK(submit_spend_ignored.m_package_feerate == std::nullopt); } // A package Package{parent1, parent2, parent3, child} where the parents are a mixture of @@ -471,10 +491,10 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) acs_witness.stack.push_back(std::vector<unsigned char>(acs_script.begin(), acs_script.end())); // parent1 will already be in the mempool - auto mtx_parent1 = CreateValidMempoolTransaction(/*input_transaction=*/ m_coinbase_txns[1], /*vout=*/ 0, - /*input_height=*/ 0, /*input_signing_key=*/ coinbaseKey, - /*output_destination=*/ acs_spk, - /*output_amount=*/ CAmount(49 * COIN), /*submit=*/ true); + auto mtx_parent1 = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[1], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/acs_spk, + /*output_amount=*/CAmount(49 * COIN), /*submit=*/true); CTransactionRef ptx_parent1 = MakeTransactionRef(mtx_parent1); package_mixed.push_back(ptx_parent1); @@ -489,10 +509,10 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) parent2_witness2.stack.push_back(std::vector<unsigned char>(grandparent2_script.begin(), grandparent2_script.end())); // Create grandparent2 creating an output with multiple spending paths. Submit to mempool. - auto mtx_grandparent2 = CreateValidMempoolTransaction(/*input_transaction=*/ m_coinbase_txns[2], /* vout=*/ 0, - /*input_height=*/ 0, /*input_signing_key=*/ coinbaseKey, - /*output_destination=*/ grandparent2_spk, - /*output_amount=*/ CAmount(49 * COIN), /*submit=*/ true); + auto mtx_grandparent2 = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[2], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/grandparent2_spk, + /*output_amount=*/CAmount(49 * COIN), /*submit=*/true); CTransactionRef ptx_grandparent2 = MakeTransactionRef(mtx_grandparent2); CMutableTransaction mtx_parent2_v1; @@ -516,11 +536,11 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) BOOST_CHECK(parent2_v2_result.m_result_type == MempoolAcceptResult::ResultType::VALID); package_mixed.push_back(ptx_parent2_v1); - // parent3 will be a new transaction - auto mtx_parent3 = CreateValidMempoolTransaction(/*input_transaction=*/ m_coinbase_txns[3], /*vout=*/ 0, - /*input_height=*/ 0, /*input_signing_key=*/ coinbaseKey, - /*output_destination=*/ acs_spk, - /*output_amount=*/ CAmount(49 * COIN), /*submit=*/ false); + // parent3 will be a new transaction. Put 0 fees on it to make it invalid on its own. + auto mtx_parent3 = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[3], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/acs_spk, + /*output_amount=*/CAmount(50 * COIN), /*submit=*/false); CTransactionRef ptx_parent3 = MakeTransactionRef(mtx_parent3); package_mixed.push_back(ptx_parent3); @@ -536,7 +556,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) mtx_mixed_child.vin[0].scriptWitness = acs_witness; mtx_mixed_child.vin[1].scriptWitness = acs_witness; mtx_mixed_child.vin[2].scriptWitness = acs_witness; - mtx_mixed_child.vout.push_back(CTxOut(145 * COIN, mixed_child_spk)); + mtx_mixed_child.vout.push_back(CTxOut((48 + 49 + 50 - 1) * COIN, mixed_child_spk)); CTransactionRef ptx_mixed_child = MakeTransactionRef(mtx_mixed_child); package_mixed.push_back(ptx_mixed_child); @@ -568,6 +588,205 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) BOOST_CHECK(!m_node.mempool->exists(GenTxid::Wtxid(ptx_parent2_v1->GetWitnessHash()))); BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent3->GetHash()))); BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_mixed_child->GetHash()))); + + // package feerate should include parent3 and child. It should not include parent1 or parent2_v1. + BOOST_CHECK(mixed_result.m_package_feerate.has_value()); + const CFeeRate expected_feerate(1 * COIN, GetVirtualTransactionSize(*ptx_parent3) + GetVirtualTransactionSize(*ptx_mixed_child)); + BOOST_CHECK_MESSAGE(mixed_result.m_package_feerate.value() == expected_feerate, + strprintf("Expected package feerate %s, got %s", expected_feerate.ToString(), + mixed_result.m_package_feerate.value().ToString())); + } +} + +BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) +{ + mineBlocks(5); + LOCK(::cs_main); + size_t expected_pool_size = m_node.mempool->size(); + CKey child_key; + child_key.MakeNewKey(true); + CScript parent_spk = GetScriptForDestination(WitnessV0KeyHash(child_key.GetPubKey())); + CKey grandchild_key; + grandchild_key.MakeNewKey(true); + CScript child_spk = GetScriptForDestination(WitnessV0KeyHash(grandchild_key.GetPubKey())); + + // zero-fee parent and high-fee child package + const CAmount coinbase_value{50 * COIN}; + const CAmount parent_value{coinbase_value - 0}; + const CAmount child_value{parent_value - COIN}; + + Package package_cpfp; + auto mtx_parent = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[0], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/parent_spk, + /*output_amount=*/parent_value, /*submit=*/false); + CTransactionRef tx_parent = MakeTransactionRef(mtx_parent); + package_cpfp.push_back(tx_parent); + + auto mtx_child = CreateValidMempoolTransaction(/*input_transaction=*/tx_parent, /*input_vout=*/0, + /*input_height=*/101, /*input_signing_key=*/child_key, + /*output_destination=*/child_spk, + /*output_amount=*/child_value, /*submit=*/false); + CTransactionRef tx_child = MakeTransactionRef(mtx_child); + package_cpfp.push_back(tx_child); + + // Package feerate is calculated using modified fees, and prioritisetransaction accepts negative + // fee deltas. This should be taken into account. De-prioritise the parent transaction by -1BTC, + // bringing the package feerate to 0. + m_node.mempool->PrioritiseTransaction(tx_parent->GetHash(), -1 * COIN); + { + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + const auto submit_cpfp_deprio = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, + package_cpfp, /*test_accept=*/ false); + BOOST_CHECK_MESSAGE(submit_cpfp_deprio.m_state.IsInvalid(), + "Package validation unexpectedly succeeded: " << submit_cpfp_deprio.m_state.GetRejectReason()); + BOOST_CHECK(submit_cpfp_deprio.m_tx_results.empty()); + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + const CFeeRate expected_feerate(0, GetVirtualTransactionSize(*tx_parent) + GetVirtualTransactionSize(*tx_child)); + BOOST_CHECK(submit_cpfp_deprio.m_package_feerate.has_value()); + BOOST_CHECK(submit_cpfp_deprio.m_package_feerate.value() == CFeeRate{0}); + BOOST_CHECK_MESSAGE(submit_cpfp_deprio.m_package_feerate.value() == expected_feerate, + strprintf("Expected package feerate %s, got %s", expected_feerate.ToString(), + submit_cpfp_deprio.m_package_feerate.value().ToString())); + } + + // Clear the prioritisation of the parent transaction. + WITH_LOCK(m_node.mempool->cs, m_node.mempool->ClearPrioritisation(tx_parent->GetHash())); + + // Package CPFP: Even though the parent pays 0 absolute fees, the child pays 1 BTC which is + // enough for the package feerate to meet the threshold. + { + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + const auto submit_cpfp = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, + package_cpfp, /*test_accept=*/ false); + expected_pool_size += 2; + BOOST_CHECK_MESSAGE(submit_cpfp.m_state.IsValid(), + "Package validation unexpectedly failed: " << submit_cpfp.m_state.GetRejectReason()); + auto it_parent = submit_cpfp.m_tx_results.find(tx_parent->GetWitnessHash()); + auto it_child = submit_cpfp.m_tx_results.find(tx_child->GetWitnessHash()); + BOOST_CHECK(it_parent != submit_cpfp.m_tx_results.end()); + BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_parent->second.m_base_fees.value() == 0); + BOOST_CHECK(it_child != submit_cpfp.m_tx_results.end()); + BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_child->second.m_base_fees.value() == COIN); + + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_parent->GetHash()))); + BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_child->GetHash()))); + + const CFeeRate expected_feerate(coinbase_value - child_value, + GetVirtualTransactionSize(*tx_parent) + GetVirtualTransactionSize(*tx_child)); + BOOST_CHECK(expected_feerate.GetFeePerK() > 1000); + BOOST_CHECK(submit_cpfp.m_package_feerate.has_value()); + BOOST_CHECK_MESSAGE(submit_cpfp.m_package_feerate.value() == expected_feerate, + strprintf("Expected package feerate %s, got %s", expected_feerate.ToString(), + submit_cpfp.m_package_feerate.value().ToString())); + } + + // Just because we allow low-fee parents doesn't mean we allow low-feerate packages. + // This package just pays 200 satoshis total. This would be enough to pay for the child alone, + // but isn't enough for the entire package to meet the 1sat/vbyte minimum. + Package package_still_too_low; + auto mtx_parent_cheap = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[1], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/parent_spk, + /*output_amount=*/coinbase_value, /*submit=*/false); + CTransactionRef tx_parent_cheap = MakeTransactionRef(mtx_parent_cheap); + package_still_too_low.push_back(tx_parent_cheap); + + auto mtx_child_cheap = CreateValidMempoolTransaction(/*input_transaction=*/tx_parent_cheap, /*input_vout=*/0, + /*input_height=*/101, /*input_signing_key=*/child_key, + /*output_destination=*/child_spk, + /*output_amount=*/coinbase_value - 200, /*submit=*/false); + CTransactionRef tx_child_cheap = MakeTransactionRef(mtx_child_cheap); + package_still_too_low.push_back(tx_child_cheap); + + // Cheap package should fail with package-fee-too-low. + { + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + const auto submit_package_too_low = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, + package_still_too_low, /*test_accept=*/false); + BOOST_CHECK_MESSAGE(submit_package_too_low.m_state.IsInvalid(), "Package validation unexpectedly succeeded"); + BOOST_CHECK_EQUAL(submit_package_too_low.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); + BOOST_CHECK_EQUAL(submit_package_too_low.m_state.GetRejectReason(), "package-fee-too-low"); + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + const CFeeRate child_feerate(200, GetVirtualTransactionSize(*tx_child_cheap)); + BOOST_CHECK(child_feerate.GetFeePerK() > 1000); + const CFeeRate expected_feerate(200, + GetVirtualTransactionSize(*tx_parent_cheap) + GetVirtualTransactionSize(*tx_child_cheap)); + BOOST_CHECK(expected_feerate.GetFeePerK() < 1000); + BOOST_CHECK(submit_package_too_low.m_package_feerate.has_value()); + BOOST_CHECK_MESSAGE(submit_package_too_low.m_package_feerate.value() == expected_feerate, + strprintf("Expected package feerate %s, got %s", expected_feerate.ToString(), + submit_package_too_low.m_package_feerate.value().ToString())); + } + + // Package feerate includes the modified fees of the transactions. + // This means a child with its fee delta from prioritisetransaction can pay for a parent. + m_node.mempool->PrioritiseTransaction(tx_child_cheap->GetHash(), 1 * COIN); + // Now that the child's fees have "increased" by 1 BTC, the cheap package should succeed. + { + const auto submit_prioritised_package = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, + package_still_too_low, /*test_accept=*/false); + expected_pool_size += 2; + BOOST_CHECK_MESSAGE(submit_prioritised_package.m_state.IsValid(), + "Package validation unexpectedly failed" << submit_prioritised_package.m_state.GetRejectReason()); + const CFeeRate expected_feerate(1 * COIN + 200, + GetVirtualTransactionSize(*tx_parent_cheap) + GetVirtualTransactionSize(*tx_child_cheap)); + BOOST_CHECK(submit_prioritised_package.m_package_feerate.has_value()); + BOOST_CHECK_MESSAGE(submit_prioritised_package.m_package_feerate.value() == expected_feerate, + strprintf("Expected package feerate %s, got %s", expected_feerate.ToString(), + submit_prioritised_package.m_package_feerate.value().ToString())); + } + + // Package feerate is calculated without topology in mind; it's just aggregating fees and sizes. + // However, this should not allow parents to pay for children. Each transaction should be + // validated individually first, eliminating sufficient-feerate parents before they are unfairly + // included in the package feerate. It's also important that the low-fee child doesn't prevent + // the parent from being accepted. + Package package_rich_parent; + const CAmount high_parent_fee{1 * COIN}; + auto mtx_parent_rich = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[2], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/parent_spk, + /*output_amount=*/coinbase_value - high_parent_fee, /*submit=*/false); + CTransactionRef tx_parent_rich = MakeTransactionRef(mtx_parent_rich); + package_rich_parent.push_back(tx_parent_rich); + + auto mtx_child_poor = CreateValidMempoolTransaction(/*input_transaction=*/tx_parent_rich, /*input_vout=*/0, + /*input_height=*/101, /*input_signing_key=*/child_key, + /*output_destination=*/child_spk, + /*output_amount=*/coinbase_value - high_parent_fee, /*submit=*/false); + CTransactionRef tx_child_poor = MakeTransactionRef(mtx_child_poor); + package_rich_parent.push_back(tx_child_poor); + + // Parent pays 1 BTC and child pays none. The parent should be accepted without the child. + { + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + const auto submit_rich_parent = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, + package_rich_parent, /*test_accept=*/false); + expected_pool_size += 1; + BOOST_CHECK_MESSAGE(submit_rich_parent.m_state.IsInvalid(), "Package validation unexpectedly succeeded"); + + // The child would have been validated on its own and failed, then submitted as a "package" of 1. + // The package feerate is just the child's feerate, which is 0sat/vb. + BOOST_CHECK(submit_rich_parent.m_package_feerate.has_value()); + BOOST_CHECK_MESSAGE(submit_rich_parent.m_package_feerate.value() == CFeeRate(), + "expected 0, got " << submit_rich_parent.m_package_feerate.value().ToString()); + BOOST_CHECK_EQUAL(submit_rich_parent.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); + BOOST_CHECK_EQUAL(submit_rich_parent.m_state.GetRejectReason(), "package-fee-too-low"); + + auto it_parent = submit_rich_parent.m_tx_results.find(tx_parent_rich->GetWitnessHash()); + BOOST_CHECK(it_parent != submit_rich_parent.m_tx_results.end()); + BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_parent->second.m_state.GetRejectReason() == ""); + BOOST_CHECK_MESSAGE(it_parent->second.m_base_fees.value() == high_parent_fee, + strprintf("rich parent: expected fee %s, got %s", high_parent_fee, it_parent->second.m_base_fees.value())); + + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_parent_rich->GetHash()))); + BOOST_CHECK(!m_node.mempool->exists(GenTxid::Txid(tx_child_poor->GetHash()))); } } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index b1e2119c64..d41b54af20 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -329,7 +329,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, Dersig100Setup) // Sign SignatureData sigdata; - BOOST_CHECK(ProduceSignature(keystore, MutableTransactionSignatureCreator(&valid_with_witness_tx, 0, 11*CENT, SIGHASH_ALL), spend_tx.vout[1].scriptPubKey, sigdata)); + BOOST_CHECK(ProduceSignature(keystore, MutableTransactionSignatureCreator(valid_with_witness_tx, 0, 11 * CENT, SIGHASH_ALL), spend_tx.vout[1].scriptPubKey, sigdata)); UpdateInput(valid_with_witness_tx.vin[0], sigdata); // This should be valid under all script flags. @@ -355,9 +355,9 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, Dersig100Setup) tx.vout[0].scriptPubKey = p2pk_scriptPubKey; // Sign - for (int i=0; i<2; ++i) { + for (int i = 0; i < 2; ++i) { SignatureData sigdata; - BOOST_CHECK(ProduceSignature(keystore, MutableTransactionSignatureCreator(&tx, i, 11*CENT, SIGHASH_ALL), spend_tx.vout[i].scriptPubKey, sigdata)); + BOOST_CHECK(ProduceSignature(keystore, MutableTransactionSignatureCreator(tx, i, 11 * CENT, SIGHASH_ALL), spend_tx.vout[i].scriptPubKey, sigdata)); UpdateInput(tx.vin[i], sigdata); } diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h index 09f96a033c..5ac504c24f 100644 --- a/src/test/util/chainstate.h +++ b/src/test/util/chainstate.h @@ -30,7 +30,7 @@ CreateAndActivateUTXOSnapshot(node::NodeContext& node, const fs::path root, F ma // int height; WITH_LOCK(::cs_main, height = node.chainman->ActiveHeight()); - fs::path snapshot_path = root / tfm::format("test_snapshot.%d.dat", height); + fs::path snapshot_path = root / fs::u8path(tfm::format("test_snapshot.%d.dat", height)); FILE* outfile{fsbridge::fopen(snapshot_path, "wb")}; CAutoFile auto_outfile{outfile, SER_DISK, CLIENT_VERSION}; diff --git a/src/test/util/logging.h b/src/test/util/logging.h index ebe0ecf623..f477088392 100644 --- a/src/test/util/logging.h +++ b/src/test/util/logging.h @@ -36,6 +36,6 @@ public: ~DebugLogHelper() { check_found(); } }; -#define ASSERT_DEBUG_LOG(message) DebugLogHelper PASTE2(debugloghelper, __COUNTER__)(message) +#define ASSERT_DEBUG_LOG(message) DebugLogHelper UNIQUE_NAME(debugloghelper)(message) #endif // BITCOIN_TEST_UTIL_LOGGING_H diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp index fe3cf52974..62b770753a 100644 --- a/src/test/util/net.cpp +++ b/src/test/util/net.cpp @@ -52,7 +52,7 @@ std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(int n_candida /*m_last_block_time=*/std::chrono::seconds{random_context.randrange(100)}, /*m_last_tx_time=*/std::chrono::seconds{random_context.randrange(100)}, /*fRelevantServices=*/random_context.randbool(), - /*fRelayTxes=*/random_context.randbool(), + /*m_relay_txs=*/random_context.randbool(), /*fBloomFilter=*/random_context.randbool(), /*nKeyedNetGroup=*/random_context.randrange(100), /*prefer_evict=*/random_context.randbool(), diff --git a/src/test/util/net.h b/src/test/util/net.h index 20c45058a1..e980fe4967 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -150,6 +150,8 @@ public: return 0; } + int SetSockOpt(int, int, const void*, socklen_t) const override { return 0; } + bool Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred = nullptr) const override diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index c968e4d124..2fc71c2a6e 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -15,10 +15,10 @@ #include <interfaces/chain.h> #include <net.h> #include <net_processing.h> -#include <node/miner.h> -#include <noui.h> #include <node/blockstorage.h> #include <node/chainstate.h> +#include <node/miner.h> +#include <noui.h> #include <policy/fees.h> #include <pow.h> #include <rpc/blockchain.h> @@ -28,6 +28,7 @@ #include <script/sigcache.h> #include <shutdown.h> #include <streams.h> +#include <test/util/net.h> #include <txdb.h> #include <util/strencodings.h> #include <util/string.h> @@ -179,11 +180,10 @@ ChainTestingSetup::~ChainTestingSetup() m_node.connman.reset(); m_node.banman.reset(); m_node.addrman.reset(); + m_node.netgroupman.reset(); m_node.args = nullptr; - UnloadBlockIndex(m_node.mempool.get(), *m_node.chainman); m_node.mempool.reset(); m_node.scheduler.reset(); - m_node.chainman->Reset(); m_node.chainman.reset(); } @@ -223,11 +223,12 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString())); } - m_node.addrman = std::make_unique<AddrMan>(/*asmap=*/std::vector<bool>(), + m_node.netgroupman = std::make_unique<NetGroupManager>(/*asmap=*/std::vector<bool>()); + m_node.addrman = std::make_unique<AddrMan>(*m_node.netgroupman, /*deterministic=*/false, m_node.args->GetIntArg("-checkaddrman", 0)); m_node.banman = std::make_unique<BanMan>(m_args.GetDataDirBase() / "banlist", nullptr, DEFAULT_MISBEHAVING_BANTIME); - m_node.connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman); // Deterministic randomness for tests. + m_node.connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman); // Deterministic randomness for tests. m_node.peerman = PeerManager::make(chainparams, *m_node.connman, *m_node.addrman, m_node.banman.get(), *m_node.chainman, *m_node.mempool, false); diff --git a/src/test/util/validation.cpp b/src/test/util/validation.cpp index 1aed492c3c..49535855f9 100644 --- a/src/test/util/validation.cpp +++ b/src/test/util/validation.cpp @@ -7,6 +7,7 @@ #include <util/check.h> #include <util/time.h> #include <validation.h> +#include <validationinterface.h> void TestChainState::ResetIbd() { @@ -20,3 +21,8 @@ void TestChainState::JumpOutOfIbd() m_cached_finished_ibd = true; Assert(!IsInitialBlockDownload()); } + +void ValidationInterfaceTest::BlockConnected(CValidationInterface& obj, const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex) +{ + obj.BlockConnected(block, pindex); +} diff --git a/src/test/util/validation.h b/src/test/util/validation.h index b13aa0be60..b0bc717b6c 100644 --- a/src/test/util/validation.h +++ b/src/test/util/validation.h @@ -7,6 +7,8 @@ #include <validation.h> +class CValidationInterface; + struct TestChainState : public CChainState { /** Reset the ibd cache to its initial state */ void ResetIbd(); @@ -14,4 +16,10 @@ struct TestChainState : public CChainState { void JumpOutOfIbd(); }; +class ValidationInterfaceTest +{ +public: + static void BlockConnected(CValidationInterface& obj, const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex); +}; + #endif // BITCOIN_TEST_UTIL_VALIDATION_H diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 9f78215de2..9dcfe009b3 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -17,6 +17,7 @@ #include <util/message.h> // For MessageSign(), MessageVerify(), MESSAGE_MAGIC #include <util/moneystr.h> #include <util/overflow.h> +#include <util/readwritefile.h> #include <util/spanparsing.h> #include <util/strencodings.h> #include <util/string.h> @@ -24,9 +25,10 @@ #include <util/vector.h> #include <array> -#include <optional> +#include <fstream> #include <limits> #include <map> +#include <optional> #include <stdint.h> #include <string.h> #include <thread> @@ -76,6 +78,31 @@ BOOST_AUTO_TEST_CASE(util_datadir) BOOST_CHECK_EQUAL(dd_norm, args.GetDataDirBase()); } +namespace { +class NoCopyOrMove +{ +public: + int i; + explicit NoCopyOrMove(int i) : i{i} { } + + NoCopyOrMove() = delete; + NoCopyOrMove(const NoCopyOrMove&) = delete; + NoCopyOrMove(NoCopyOrMove&&) = delete; + NoCopyOrMove& operator=(const NoCopyOrMove&) = delete; + NoCopyOrMove& operator=(NoCopyOrMove&&) = delete; + + operator bool() const { return i != 0; } + + int get_ip1() { return i + 1; } + bool test() + { + // Check that Assume can be used within a lambda and still call methods + [&]() { Assume(get_ip1()); }(); + return Assume(get_ip1() != 5); + } +}; +} // namespace + BOOST_AUTO_TEST_CASE(util_check) { // Check that Assert can forward @@ -87,6 +114,14 @@ BOOST_AUTO_TEST_CASE(util_check) // Check that Assume can be used as unary expression const bool result{Assume(two == 2)}; Assert(result); + + // Check that Assert doesn't require copy/move + NoCopyOrMove x{9}; + Assert(x).i += 3; + Assert(x).test(); + + // Check nested Asserts + BOOST_CHECK_EQUAL(Assert((Assert(x).test() ? 3 : 0)), 3); } BOOST_AUTO_TEST_CASE(util_criticalsection) @@ -163,6 +198,24 @@ BOOST_AUTO_TEST_CASE(util_HexStr) BOOST_CHECK_EQUAL(HexStr(in_s), out_exp); BOOST_CHECK_EQUAL(HexStr(in_b), out_exp); } + + { + auto input = std::string(); + for (size_t i=0; i<256; ++i) { + input.push_back(static_cast<char>(i)); + } + + auto hex = HexStr(input); + BOOST_TEST_REQUIRE(hex.size() == 512); + static constexpr auto hexmap = std::string_view("0123456789abcdef"); + for (size_t i = 0; i < 256; ++i) { + auto upper = hexmap.find(hex[i * 2]); + auto lower = hexmap.find(hex[i * 2 + 1]); + BOOST_TEST_REQUIRE(upper != std::string_view::npos); + BOOST_TEST_REQUIRE(lower != std::string_view::npos); + BOOST_TEST_REQUIRE(i == upper*16 + lower); + } + } } BOOST_AUTO_TEST_CASE(span_write_bytes) @@ -191,17 +244,17 @@ BOOST_AUTO_TEST_CASE(util_Join) BOOST_AUTO_TEST_CASE(util_TrimString) { BOOST_CHECK_EQUAL(TrimString(" foo bar "), "foo bar"); - BOOST_CHECK_EQUAL(TrimString("\t \n \n \f\n\r\t\v\tfoo \n \f\n\r\t\v\tbar\t \n \f\n\r\t\v\t\n "), "foo \n \f\n\r\t\v\tbar"); + BOOST_CHECK_EQUAL(TrimStringView("\t \n \n \f\n\r\t\v\tfoo \n \f\n\r\t\v\tbar\t \n \f\n\r\t\v\t\n "), "foo \n \f\n\r\t\v\tbar"); BOOST_CHECK_EQUAL(TrimString("\t \n foo \n\tbar\t \n "), "foo \n\tbar"); - BOOST_CHECK_EQUAL(TrimString("\t \n foo \n\tbar\t \n ", "fobar"), "\t \n foo \n\tbar\t \n "); + BOOST_CHECK_EQUAL(TrimStringView("\t \n foo \n\tbar\t \n ", "fobar"), "\t \n foo \n\tbar\t \n "); BOOST_CHECK_EQUAL(TrimString("foo bar"), "foo bar"); - BOOST_CHECK_EQUAL(TrimString("foo bar", "fobar"), " "); + BOOST_CHECK_EQUAL(TrimStringView("foo bar", "fobar"), " "); BOOST_CHECK_EQUAL(TrimString(std::string("\0 foo \0 ", 8)), std::string("\0 foo \0", 7)); - BOOST_CHECK_EQUAL(TrimString(std::string(" foo ", 5)), std::string("foo", 3)); + BOOST_CHECK_EQUAL(TrimStringView(std::string(" foo ", 5)), std::string("foo", 3)); BOOST_CHECK_EQUAL(TrimString(std::string("\t\t\0\0\n\n", 6)), std::string("\0\0", 2)); - BOOST_CHECK_EQUAL(TrimString(std::string("\x05\x04\x03\x02\x01\x00", 6)), std::string("\x05\x04\x03\x02\x01\x00", 6)); + BOOST_CHECK_EQUAL(TrimStringView(std::string("\x05\x04\x03\x02\x01\x00", 6)), std::string("\x05\x04\x03\x02\x01\x00", 6)); BOOST_CHECK_EQUAL(TrimString(std::string("\x05\x04\x03\x02\x01\x00", 6), std::string("\x05\x04\x03\x02\x01", 5)), std::string("\0", 1)); - BOOST_CHECK_EQUAL(TrimString(std::string("\x05\x04\x03\x02\x01\x00", 6), std::string("\x05\x04\x03\x02\x01\x00", 6)), ""); + BOOST_CHECK_EQUAL(TrimStringView(std::string("\x05\x04\x03\x02\x01\x00", 6), std::string("\x05\x04\x03\x02\x01\x00", 6)), ""); } BOOST_AUTO_TEST_CASE(util_FormatParseISO8601DateTime) @@ -1435,8 +1488,8 @@ BOOST_AUTO_TEST_CASE(util_time_GetTime) { SetMockTime(111); // Check that mock time does not change after a sleep - for (const auto& num_sleep : {0, 1}) { - UninterruptibleSleep(std::chrono::milliseconds{num_sleep}); + for (const auto& num_sleep : {0ms, 1ms}) { + UninterruptibleSleep(num_sleep); BOOST_CHECK_EQUAL(111, GetTime()); // Deprecated time getter BOOST_CHECK_EQUAL(111, GetTime<std::chrono::seconds>().count()); BOOST_CHECK_EQUAL(111000, GetTime<std::chrono::milliseconds>().count()); @@ -1444,10 +1497,14 @@ BOOST_AUTO_TEST_CASE(util_time_GetTime) } SetMockTime(0); - // Check that system time changes after a sleep + // Check that steady time and system time changes after a sleep + const auto steady_ms_0 = Now<SteadyMilliseconds>(); + const auto steady_0 = std::chrono::steady_clock::now(); const auto ms_0 = GetTime<std::chrono::milliseconds>(); const auto us_0 = GetTime<std::chrono::microseconds>(); - UninterruptibleSleep(std::chrono::milliseconds{1}); + UninterruptibleSleep(1ms); + BOOST_CHECK(steady_ms_0 < Now<SteadyMilliseconds>()); + BOOST_CHECK(steady_0 + 1ms <= std::chrono::steady_clock::now()); BOOST_CHECK(ms_0 < GetTime<std::chrono::milliseconds>()); BOOST_CHECK(us_0 < GetTime<std::chrono::microseconds>()); } @@ -2014,7 +2071,7 @@ BOOST_AUTO_TEST_CASE(test_ParseFixedPoint) BOOST_CHECK(!ParseFixedPoint("31.999999999999999999999", 3, &amount)); } -static void TestOtherThread(fs::path dirname, std::string lockname, bool *result) +static void TestOtherThread(fs::path dirname, fs::path lockname, bool *result) { *result = LockDirectory(dirname, lockname); } @@ -2024,7 +2081,7 @@ static constexpr char LockCommand = 'L'; static constexpr char UnlockCommand = 'U'; static constexpr char ExitCommand = 'X'; -[[noreturn]] static void TestOtherProcess(fs::path dirname, std::string lockname, int fd) +[[noreturn]] static void TestOtherProcess(fs::path dirname, fs::path lockname, int fd) { char ch; while (true) { @@ -2055,7 +2112,7 @@ static constexpr char ExitCommand = 'X'; BOOST_AUTO_TEST_CASE(test_LockDirectory) { fs::path dirname = m_args.GetDataDirBase() / "lock_dir"; - const std::string lockname = ".lock"; + const fs::path lockname = ".lock"; #ifndef WIN32 // Revert SIGCHLD to default, otherwise boost.test will catch and fail on // it: there is BOOST_TEST_IGNORE_SIGCHLD but that only works when defined @@ -2314,6 +2371,68 @@ BOOST_AUTO_TEST_CASE(test_spanparsing) BOOST_CHECK_EQUAL(SpanToStr(results[3]), ""); } +BOOST_AUTO_TEST_CASE(test_SplitString) +{ + // Empty string. + { + std::vector<std::string> result = SplitString("", '-'); + BOOST_CHECK_EQUAL(result.size(), 1); + BOOST_CHECK_EQUAL(result[0], ""); + } + + // Empty items. + { + std::vector<std::string> result = SplitString("-", '-'); + BOOST_CHECK_EQUAL(result.size(), 2); + BOOST_CHECK_EQUAL(result[0], ""); + BOOST_CHECK_EQUAL(result[1], ""); + } + + // More empty items. + { + std::vector<std::string> result = SplitString("--", '-'); + BOOST_CHECK_EQUAL(result.size(), 3); + BOOST_CHECK_EQUAL(result[0], ""); + BOOST_CHECK_EQUAL(result[1], ""); + BOOST_CHECK_EQUAL(result[2], ""); + } + + // Separator is not present. + { + std::vector<std::string> result = SplitString("abc", '-'); + BOOST_CHECK_EQUAL(result.size(), 1); + BOOST_CHECK_EQUAL(result[0], "abc"); + } + + // Basic behavior. + { + std::vector<std::string> result = SplitString("a-b", '-'); + BOOST_CHECK_EQUAL(result.size(), 2); + BOOST_CHECK_EQUAL(result[0], "a"); + BOOST_CHECK_EQUAL(result[1], "b"); + } + + // Case-sensitivity of the separator. + { + std::vector<std::string> result = SplitString("AAA", 'a'); + BOOST_CHECK_EQUAL(result.size(), 1); + BOOST_CHECK_EQUAL(result[0], "AAA"); + } + + // multiple split characters + { + using V = std::vector<std::string>; + BOOST_TEST(SplitString("a,b.c:d;e", ",;") == V({"a", "b.c:d", "e"})); + BOOST_TEST(SplitString("a,b.c:d;e", ",;:.") == V({"a", "b", "c", "d", "e"})); + BOOST_TEST(SplitString("a,b.c:d;e", "") == V({"a,b.c:d;e"})); + BOOST_TEST(SplitString("aaa", "bcdefg") == V({"aaa"})); + BOOST_TEST(SplitString("x\0a,b"s, "\0"s) == V({"x", "a,b"})); + BOOST_TEST(SplitString("x\0a,b"s, '\0') == V({"x", "a,b"})); + BOOST_TEST(SplitString("x\0a,b"s, "\0,"s) == V({"x", "a", "b"})); + BOOST_TEST(SplitString("abcdefg", "bcd") == V({"a", "", "", "efg"})); + } +} + BOOST_AUTO_TEST_CASE(test_LogEscapeMessage) { // ASCII and UTF-8 must pass through unaltered. @@ -2534,13 +2653,13 @@ BOOST_AUTO_TEST_CASE(message_hash) BOOST_AUTO_TEST_CASE(remove_prefix) { BOOST_CHECK_EQUAL(RemovePrefix("./util/system.h", "./"), "util/system.h"); - BOOST_CHECK_EQUAL(RemovePrefix("foo", "foo"), ""); + BOOST_CHECK_EQUAL(RemovePrefixView("foo", "foo"), ""); BOOST_CHECK_EQUAL(RemovePrefix("foo", "fo"), "o"); - BOOST_CHECK_EQUAL(RemovePrefix("foo", "f"), "oo"); + BOOST_CHECK_EQUAL(RemovePrefixView("foo", "f"), "oo"); BOOST_CHECK_EQUAL(RemovePrefix("foo", ""), "foo"); - BOOST_CHECK_EQUAL(RemovePrefix("fo", "foo"), "fo"); + BOOST_CHECK_EQUAL(RemovePrefixView("fo", "foo"), "fo"); BOOST_CHECK_EQUAL(RemovePrefix("f", "foo"), "f"); - BOOST_CHECK_EQUAL(RemovePrefix("", "foo"), ""); + BOOST_CHECK_EQUAL(RemovePrefixView("", "foo"), ""); BOOST_CHECK_EQUAL(RemovePrefix("", ""), ""); } @@ -2592,4 +2711,49 @@ BOOST_AUTO_TEST_CASE(util_ParseByteUnits) BOOST_CHECK(!ParseByteUnits("1x", noop)); } +BOOST_AUTO_TEST_CASE(util_ReadBinaryFile) +{ + fs::path tmpfolder = m_args.GetDataDirBase(); + fs::path tmpfile = tmpfolder / "read_binary.dat"; + std::string expected_text; + for (int i = 0; i < 30; i++) { + expected_text += "0123456789"; + } + { + std::ofstream file{tmpfile}; + file << expected_text; + } + { + // read all contents in file + auto [valid, text] = ReadBinaryFile(tmpfile); + BOOST_CHECK(valid); + BOOST_CHECK_EQUAL(text, expected_text); + } + { + // read half contents in file + auto [valid, text] = ReadBinaryFile(tmpfile, expected_text.size() / 2); + BOOST_CHECK(valid); + BOOST_CHECK_EQUAL(text, expected_text.substr(0, expected_text.size() / 2)); + } + { + // read from non-existent file + fs::path invalid_file = tmpfolder / "invalid_binary.dat"; + auto [valid, text] = ReadBinaryFile(invalid_file); + BOOST_CHECK(!valid); + BOOST_CHECK(text.empty()); + } +} + +BOOST_AUTO_TEST_CASE(util_WriteBinaryFile) +{ + fs::path tmpfolder = m_args.GetDataDirBase(); + fs::path tmpfile = tmpfolder / "write_binary.dat"; + std::string expected_text = "bitcoin"; + auto valid = WriteBinaryFile(tmpfile, expected_text); + std::string actual_text; + std::ifstream file{tmpfile}; + file >> actual_text; + BOOST_CHECK(valid); + BOOST_CHECK_EQUAL(actual_text, expected_text); +} BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 1beef5cf04..2a3990bb7c 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -72,9 +72,6 @@ BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) // The view cache should be empty since we had to destruct to downsize. BOOST_CHECK(!c1.CoinsTip().HaveCoinInCache(outpoint)); } - - // Avoid triggering the address sanitizer. - WITH_LOCK(::cs_main, manager.Unload()); } //! Test UpdateTip behavior for both active and background chainstates. @@ -96,7 +93,7 @@ BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(m_node, m_path_root)); // Ensure our active chain is the snapshot chainstate. - BOOST_CHECK(chainman.IsSnapshotActive()); + BOOST_CHECK(WITH_LOCK(::cs_main, return chainman.IsSnapshotActive())); curr_tip = ::g_best_block; diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 26392e690d..6dc522b421 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -45,7 +45,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23)); BOOST_CHECK(!manager.IsSnapshotActive()); - BOOST_CHECK(!manager.IsSnapshotValidated()); + BOOST_CHECK(WITH_LOCK(::cs_main, return !manager.IsSnapshotValidated())); auto all = manager.GetAll(); BOOST_CHECK_EQUAL_COLLECTIONS(all.begin(), all.end(), chainstates.begin(), chainstates.end()); @@ -78,7 +78,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) BOOST_CHECK(c2.ActivateBestChain(_, nullptr)); BOOST_CHECK(manager.IsSnapshotActive()); - BOOST_CHECK(!manager.IsSnapshotValidated()); + BOOST_CHECK(WITH_LOCK(::cs_main, return !manager.IsSnapshotValidated())); BOOST_CHECK_EQUAL(&c2, &manager.ActiveChainstate()); BOOST_CHECK(&c1 != &manager.ActiveChainstate()); auto all2 = manager.GetAll(); @@ -99,8 +99,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) // Let scheduler events finish running to avoid accessing memory that is going to be unloaded SyncWithValidationInterfaceQueue(); - - WITH_LOCK(::cs_main, manager.Unload()); } //! Test rebalancing the caches associated with each chainstate. diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 7ae384ceb3..05dbc6057f 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -24,10 +24,6 @@ #include <set> #include <vector> -#include <boost/algorithm/string/classification.hpp> -#include <boost/algorithm/string/replace.hpp> -#include <boost/algorithm/string/split.hpp> - #include <event2/buffer.h> #include <event2/bufferevent.h> #include <event2/event.h> @@ -53,6 +49,7 @@ static const float RECONNECT_TIMEOUT_EXP = 1.5; * this is belt-and-suspenders sanity limit to prevent memory exhaustion. */ static const int MAX_LINE_LENGTH = 100000; +static const uint16_t DEFAULT_TOR_SOCKS_PORT = 9050; /****** Low-level TorControlConnection ********/ @@ -307,7 +304,7 @@ std::map<std::string,std::string> ParseTorReplyMapping(const std::string &s) TorController::TorController(struct event_base* _base, const std::string& tor_control_center, const CService& target): base(_base), - m_tor_control_center(tor_control_center), conn(base), reconnect(true), reconnect_ev(0), + m_tor_control_center(tor_control_center), conn(base), reconnect(true), reconnect_ev(nullptr), reconnect_timeout(RECONNECT_TIMEOUT_START), m_target(target) { @@ -338,6 +335,73 @@ TorController::~TorController() } } +void TorController::get_socks_cb(TorControlConnection& _conn, const TorControlReply& reply) +{ + // NOTE: We can only get here if -onion is unset + std::string socks_location; + if (reply.code == 250) { + for (const auto& line : reply.lines) { + if (0 == line.compare(0, 20, "net/listeners/socks=")) { + const std::string port_list_str = line.substr(20); + std::vector<std::string> port_list = SplitString(port_list_str, ' '); + + for (auto& portstr : port_list) { + if (portstr.empty()) continue; + if ((portstr[0] == '"' || portstr[0] == '\'') && portstr.size() >= 2 && (*portstr.rbegin() == portstr[0])) { + portstr = portstr.substr(1, portstr.size() - 2); + if (portstr.empty()) continue; + } + socks_location = portstr; + if (0 == portstr.compare(0, 10, "127.0.0.1:")) { + // Prefer localhost - ignore other ports + break; + } + } + } + } + if (!socks_location.empty()) { + LogPrint(BCLog::TOR, "tor: Get SOCKS port command yielded %s\n", socks_location); + } else { + LogPrintf("tor: Get SOCKS port command returned nothing\n"); + } + } else if (reply.code == 510) { // 510 Unrecognized command + LogPrintf("tor: Get SOCKS port command failed with unrecognized command (You probably should upgrade Tor)\n"); + } else { + LogPrintf("tor: Get SOCKS port command failed; error code %d\n", reply.code); + } + + CService resolved; + Assume(!resolved.IsValid()); + if (!socks_location.empty()) { + resolved = LookupNumeric(socks_location, DEFAULT_TOR_SOCKS_PORT); + } + if (!resolved.IsValid()) { + // Fallback to old behaviour + resolved = LookupNumeric("127.0.0.1", DEFAULT_TOR_SOCKS_PORT); + } + + Assume(resolved.IsValid()); + LogPrint(BCLog::TOR, "tor: Configuring onion proxy for %s\n", resolved.ToStringIPPort()); + Proxy addrOnion = Proxy(resolved, true); + SetProxy(NET_ONION, addrOnion); + + const auto onlynets = gArgs.GetArgs("-onlynet"); + + const bool onion_allowed_by_onlynet{ + !gArgs.IsArgSet("-onlynet") || + std::any_of(onlynets.begin(), onlynets.end(), [](const auto& n) { + return ParseNetwork(n) == NET_ONION; + })}; + + if (onion_allowed_by_onlynet) { + // If NET_ONION is reachable, then the below is a noop. + // + // If NET_ONION is not reachable, then none of -proxy or -onion was given. + // Since we are here, then -torcontrol and -torpassword were given. + SetReachable(NET_ONION, true); + } +} + void TorController::add_onion_cb(TorControlConnection& _conn, const TorControlReply& reply) { if (reply.code == 250) { @@ -381,25 +445,7 @@ void TorController::auth_cb(TorControlConnection& _conn, const TorControlReply& // Now that we know Tor is running setup the proxy for onion addresses // if -onion isn't set to something else. if (gArgs.GetArg("-onion", "") == "") { - CService resolved(LookupNumeric("127.0.0.1", 9050)); - Proxy addrOnion = Proxy(resolved, true); - SetProxy(NET_ONION, addrOnion); - - const auto onlynets = gArgs.GetArgs("-onlynet"); - - const bool onion_allowed_by_onlynet{ - !gArgs.IsArgSet("-onlynet") || - std::any_of(onlynets.begin(), onlynets.end(), [](const auto& n) { - return ParseNetwork(n) == NET_ONION; - })}; - - if (onion_allowed_by_onlynet) { - // If NET_ONION is reachable, then the below is a noop. - // - // If NET_ONION is not reachable, then none of -proxy or -onion was given. - // Since we are here, then -torcontrol and -torpassword were given. - SetReachable(NET_ONION, true); - } + _conn.Command("GETINFO net/listeners/socks", std::bind(&TorController::get_socks_cb, this, std::placeholders::_1, std::placeholders::_2)); } // Finally - now create the service @@ -492,8 +538,10 @@ void TorController::protocolinfo_cb(TorControlConnection& _conn, const TorContro if (l.first == "AUTH") { std::map<std::string,std::string> m = ParseTorReplyMapping(l.second); std::map<std::string,std::string>::iterator i; - if ((i = m.find("METHODS")) != m.end()) - boost::split(methods, i->second, boost::is_any_of(",")); + if ((i = m.find("METHODS")) != m.end()) { + std::vector<std::string> m_vec = SplitString(i->second, ','); + methods = std::set<std::string>(m_vec.begin(), m_vec.end()); + } if ((i = m.find("COOKIEFILE")) != m.end()) cookiefile = i->second; } else if (l.first == "VERSION") { @@ -516,7 +564,7 @@ void TorController::protocolinfo_cb(TorControlConnection& _conn, const TorContro if (!torpassword.empty()) { if (methods.count("HASHEDPASSWORD")) { LogPrint(BCLog::TOR, "tor: Using HASHEDPASSWORD authentication\n"); - boost::replace_all(torpassword, "\"", "\\\""); + ReplaceAll(torpassword, "\"", "\\\""); _conn.Command("AUTHENTICATE \"" + torpassword + "\"", std::bind(&TorController::auth_cb, this, std::placeholders::_1, std::placeholders::_2)); } else { LogPrintf("tor: Password provided with -torpassword, but HASHEDPASSWORD authentication is not available\n"); @@ -532,7 +580,7 @@ void TorController::protocolinfo_cb(TorControlConnection& _conn, const TorContro // _conn.Command("AUTHENTICATE " + HexStr(status_cookie.second), std::bind(&TorController::auth_cb, this, std::placeholders::_1, std::placeholders::_2)); cookie = std::vector<uint8_t>(status_cookie.second.begin(), status_cookie.second.end()); clientNonce = std::vector<uint8_t>(TOR_NONCE_SIZE, 0); - GetRandBytes(clientNonce.data(), TOR_NONCE_SIZE); + GetRandBytes(clientNonce); _conn.Command("AUTHCHALLENGE SAFECOOKIE " + HexStr(clientNonce), std::bind(&TorController::authchallenge_cb, this, std::placeholders::_1, std::placeholders::_2)); } else { if (status_cookie.first) { diff --git a/src/torcontrol.h b/src/torcontrol.h index 4ace3edcb1..81475aee74 100644 --- a/src/torcontrol.h +++ b/src/torcontrol.h @@ -140,6 +140,8 @@ private: std::vector<uint8_t> clientNonce; public: + /** Callback for GETINFO net/listeners/socks result */ + void get_socks_cb(TorControlConnection& conn, const TorControlReply& reply); /** Callback for ADD_ONION result */ void add_onion_cb(TorControlConnection& conn, const TorControlReply& reply); /** Callback for AUTHENTICATE result */ diff --git a/src/txdb.cpp b/src/txdb.cpp index 9a39e90ccd..afcd1985f5 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -6,7 +6,6 @@ #include <txdb.h> #include <chain.h> -#include <node/ui_interface.h> #include <pow.h> #include <random.h> #include <shutdown.h> @@ -18,7 +17,6 @@ #include <stdint.h> static constexpr uint8_t DB_COIN{'C'}; -static constexpr uint8_t DB_COINS{'c'}; static constexpr uint8_t DB_BLOCK_FILES{'f'}; static constexpr uint8_t DB_BLOCK_INDEX{'b'}; @@ -29,6 +27,7 @@ static constexpr uint8_t DB_REINDEX_FLAG{'R'}; static constexpr uint8_t DB_LAST_BLOCK{'l'}; // Keys used in previous version that might still be found in the DB: +static constexpr uint8_t DB_COINS{'c'}; static constexpr uint8_t DB_TXINDEX_BLOCK{'T'}; // uint8_t DB_TXINDEX{'t'} @@ -50,6 +49,15 @@ std::optional<bilingual_str> CheckLegacyTxindex(CBlockTreeDB& block_tree_db) return std::nullopt; } +bool CCoinsViewDB::NeedsUpgrade() +{ + std::unique_ptr<CDBIterator> cursor{m_db->NewIterator()}; + // DB_COINS was deprecated in v0.15.0, commit + // 1088b02f0ccd7358d2b7076bb9e122d59d502d02 + cursor->Seek(std::make_pair(DB_COINS, uint256{})); + return cursor->Valid(); +} + namespace { struct CoinEntry { @@ -60,7 +68,7 @@ struct CoinEntry { SERIALIZE_METHODS(CoinEntry, obj) { READWRITE(obj.key, obj.outpoint->hash, VARINT(obj.outpoint->n)); } }; -} +} // namespace CCoinsViewDB::CCoinsViewDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe) : m_db(std::make_unique<CDBWrapper>(ldb_path, nCacheSize, fMemory, fWipe, true)), @@ -76,7 +84,7 @@ void CCoinsViewDB::ResizeCache(size_t new_cache_size) // filesystem lock. m_db.reset(); m_db = std::make_unique<CDBWrapper>( - m_ldb_path, new_cache_size, m_is_memory, /*fWipe*/ false, /*obfuscate*/ true); + m_ldb_path, new_cache_size, m_is_memory, /*fWipe=*/false, /*obfuscate=*/true); } } @@ -337,125 +345,3 @@ bool CBlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, return true; } - -namespace { - -//! Legacy class to deserialize pre-pertxout database entries without reindex. -class CCoins -{ -public: - //! whether transaction is a coinbase - bool fCoinBase; - - //! unspent transaction outputs; spent outputs are .IsNull(); spent outputs at the end of the array are dropped - std::vector<CTxOut> vout; - - //! at which height this transaction was included in the active block chain - int nHeight; - - //! empty constructor - CCoins() : fCoinBase(false), vout(0), nHeight(0) { } - - template<typename Stream> - void Unserialize(Stream &s) { - unsigned int nCode = 0; - // version - unsigned int nVersionDummy; - ::Unserialize(s, VARINT(nVersionDummy)); - // header code - ::Unserialize(s, VARINT(nCode)); - fCoinBase = nCode & 1; - std::vector<bool> vAvail(2, false); - vAvail[0] = (nCode & 2) != 0; - vAvail[1] = (nCode & 4) != 0; - unsigned int nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1); - // spentness bitmask - while (nMaskCode > 0) { - unsigned char chAvail = 0; - ::Unserialize(s, chAvail); - for (unsigned int p = 0; p < 8; p++) { - bool f = (chAvail & (1 << p)) != 0; - vAvail.push_back(f); - } - if (chAvail != 0) - nMaskCode--; - } - // txouts themself - vout.assign(vAvail.size(), CTxOut()); - for (unsigned int i = 0; i < vAvail.size(); i++) { - if (vAvail[i]) - ::Unserialize(s, Using<TxOutCompression>(vout[i])); - } - // coinbase height - ::Unserialize(s, VARINT_MODE(nHeight, VarIntMode::NONNEGATIVE_SIGNED)); - } -}; - -} - -/** Upgrade the database from older formats. - * - * Currently implemented: from the per-tx utxo model (0.8..0.14.x) to per-txout. - */ -bool CCoinsViewDB::Upgrade() { - std::unique_ptr<CDBIterator> pcursor(m_db->NewIterator()); - pcursor->Seek(std::make_pair(DB_COINS, uint256())); - if (!pcursor->Valid()) { - return true; - } - - int64_t count = 0; - LogPrintf("Upgrading utxo-set database...\n"); - LogPrintf("[0%%]..."); /* Continued */ - uiInterface.ShowProgress(_("Upgrading UTXO database").translated, 0, true); - size_t batch_size = 1 << 24; - CDBBatch batch(*m_db); - int reportDone = 0; - std::pair<unsigned char, uint256> key; - std::pair<unsigned char, uint256> prev_key = {DB_COINS, uint256()}; - while (pcursor->Valid()) { - if (ShutdownRequested()) { - break; - } - if (pcursor->GetKey(key) && key.first == DB_COINS) { - if (count++ % 256 == 0) { - uint32_t high = 0x100 * *key.second.begin() + *(key.second.begin() + 1); - int percentageDone = (int)(high * 100.0 / 65536.0 + 0.5); - uiInterface.ShowProgress(_("Upgrading UTXO database").translated, percentageDone, true); - if (reportDone < percentageDone/10) { - // report max. every 10% step - LogPrintf("[%d%%]...", percentageDone); /* Continued */ - reportDone = percentageDone/10; - } - } - CCoins old_coins; - if (!pcursor->GetValue(old_coins)) { - return error("%s: cannot parse CCoins record", __func__); - } - COutPoint outpoint(key.second, 0); - for (size_t i = 0; i < old_coins.vout.size(); ++i) { - if (!old_coins.vout[i].IsNull() && !old_coins.vout[i].scriptPubKey.IsUnspendable()) { - Coin newcoin(std::move(old_coins.vout[i]), old_coins.nHeight, old_coins.fCoinBase); - outpoint.n = i; - CoinEntry entry(&outpoint); - batch.Write(entry, newcoin); - } - } - batch.Erase(key); - if (batch.SizeEstimate() > batch_size) { - m_db->WriteBatch(batch); - batch.Clear(); - m_db->CompactRange(prev_key, key); - prev_key = key; - } - pcursor->Next(); - } else { - break; - } - } - m_db->WriteBatch(batch); - m_db->CompactRange({DB_COINS, uint256()}, key); - uiInterface.ShowProgress("", 100, false); - LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE"); - return !ShutdownRequested(); -} diff --git a/src/txdb.h b/src/txdb.h index e70f3cd1f2..faa543b412 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -65,8 +65,8 @@ public: bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override; std::unique_ptr<CCoinsViewCursor> Cursor() const override; - //! Attempt to update from an older database format. Returns whether an error occurred. - bool Upgrade(); + //! Whether an unsupported database format is used. + bool NeedsUpgrade(); size_t EstimateSize() const override; //! Dynamically alter the underlying leveldb cache size. diff --git a/src/txmempool.cpp b/src/txmempool.cpp index fb5652d0a0..65c8b4ea60 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -54,16 +54,6 @@ struct update_ancestor_state int64_t modifySigOpsCost; }; -struct update_fee_delta -{ - explicit update_fee_delta(int64_t _feeDelta) : feeDelta(_feeDelta) { } - - void operator() (CTxMemPoolEntry &e) { e.UpdateFeeDelta(feeDelta); } - -private: - int64_t feeDelta; -}; - bool TestLockPointValidity(CChain& active_chain, const LockPoints& lp) { AssertLockHeld(cs_main); @@ -99,7 +89,7 @@ CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& tx, CAmount fee, nModFeesWithAncestors{nFee}, nSigOpCostWithAncestors{sigOpCost} {} -void CTxMemPoolEntry::UpdateFeeDelta(int64_t newFeeDelta) +void CTxMemPoolEntry::UpdateFeeDelta(CAmount newFeeDelta) { nModFeesWithDescendants += newFeeDelta - feeDelta; nModFeesWithAncestors += newFeeDelta - feeDelta; @@ -338,7 +328,7 @@ bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, staged_ancestors = it->GetMemPoolParentsConst(); } - return CalculateAncestorsAndCheckLimits(entry.GetTxSize(), /* entry_count */ 1, + return CalculateAncestorsAndCheckLimits(entry.GetTxSize(), /*entry_count=*/1, setAncestors, staged_ancestors, limitAncestorCount, limitAncestorSize, limitDescendantCount, limitDescendantSize, errString); @@ -491,12 +481,10 @@ void CTxMemPool::addUnchecked(const CTxMemPoolEntry &entry, setEntries &setAnces indexed_transaction_set::iterator newit = mapTx.insert(entry).first; // Update transaction for any feeDelta created by PrioritiseTransaction - // TODO: refactor so that the fee delta is calculated before inserting - // into mapTx. CAmount delta{0}; ApplyDelta(entry.GetTx().GetHash(), delta); if (delta) { - mapTx.modify(newit, update_fee_delta(delta)); + mapTx.modify(newit, [&delta](CTxMemPoolEntry& e) { e.UpdateFeeDelta(delta); }); } // Update cachedInnerUsage to include contained transaction's usage. @@ -704,6 +692,7 @@ void CTxMemPool::removeForBlock(const std::vector<CTransactionRef>& vtx, unsigne void CTxMemPool::_clear() { + vTxHashes.clear(); mapTx.clear(); mapNextTx.clear(); totalTxSize = 0; @@ -931,7 +920,7 @@ void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeD delta += nFeeDelta; txiter it = mapTx.find(hash); if (it != mapTx.end()) { - mapTx.modify(it, update_fee_delta(delta)); + mapTx.modify(it, [&delta](CTxMemPoolEntry& e) { e.UpdateFeeDelta(delta); }); // Now update all ancestors' modified fees with descendants setEntries setAncestors; uint64_t nNoLimit = std::numeric_limits<uint64_t>::max(); diff --git a/src/txmempool.h b/src/txmempool.h index e7e5a3c402..f5d5abc62e 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -101,7 +101,7 @@ private: const unsigned int entryHeight; //!< Chain height when entering the mempool const bool spendsCoinbase; //!< keep track of transactions that spend a coinbase const int64_t sigOpCost; //!< Total sigop cost - int64_t feeDelta{0}; //!< Used for determining the priority of the transaction for mining in a block + CAmount feeDelta{0}; //!< Used for determining the priority of the transaction for mining in a block LockPoints lockPoints; //!< Track the height and time at which tx was final // Information about descendants of this transaction that are in the @@ -131,7 +131,7 @@ public: std::chrono::seconds GetTime() const { return std::chrono::seconds{nTime}; } unsigned int GetHeight() const { return entryHeight; } int64_t GetSigOpCost() const { return sigOpCost; } - int64_t GetModifiedFee() const { return nFee + feeDelta; } + CAmount GetModifiedFee() const { return nFee + feeDelta; } size_t DynamicMemoryUsage() const { return nUsageSize; } const LockPoints& GetLockPoints() const { return lockPoints; } @@ -140,8 +140,8 @@ public: // Adjusts the ancestor state void UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps); // Updates the fee delta used for mining priority score, and the - // modified fees with descendants. - void UpdateFeeDelta(int64_t feeDelta); + // modified fees with descendants/ancestors. + void UpdateFeeDelta(CAmount newFeeDelta); // Update the LockPoints after a reorg void UpdateLockPoints(const LockPoints& lp); diff --git a/src/util/bytevectorhash.cpp b/src/util/bytevectorhash.cpp index f87d0e04b3..bc060a44c9 100644 --- a/src/util/bytevectorhash.cpp +++ b/src/util/bytevectorhash.cpp @@ -8,8 +8,8 @@ ByteVectorHash::ByteVectorHash() { - GetRandBytes(reinterpret_cast<unsigned char*>(&m_k0), sizeof(m_k0)); - GetRandBytes(reinterpret_cast<unsigned char*>(&m_k1), sizeof(m_k1)); + GetRandBytes({reinterpret_cast<unsigned char*>(&m_k0), sizeof(m_k0)}); + GetRandBytes({reinterpret_cast<unsigned char*>(&m_k1), sizeof(m_k1)}); } size_t ByteVectorHash::operator()(const std::vector<unsigned char>& input) const diff --git a/src/util/check.cpp b/src/util/check.cpp new file mode 100644 index 0000000000..2a9f885560 --- /dev/null +++ b/src/util/check.cpp @@ -0,0 +1,14 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/check.h> + +#include <tinyformat.h> + +void assertion_fail(const char* file, int line, const char* func, const char* assertion) +{ + auto str = strprintf("%s:%s %s: Assertion `%s' failed.\n", file, line, func, assertion); + fwrite(str.data(), 1, str.size(), stderr); + std::abort(); +} diff --git a/src/util/check.h b/src/util/check.h index a443c13cf2..aca957925a 100644 --- a/src/util/check.h +++ b/src/util/check.h @@ -18,8 +18,23 @@ class NonFatalCheckError : public std::runtime_error using std::runtime_error::runtime_error; }; +#define format_internal_error(msg, file, line, func, report) \ + strprintf("Internal bug detected: \"%s\"\n%s:%d (%s)\nPlease report this issue here: %s\n", \ + msg, file, line, func, report) + +/** Helper for CHECK_NONFATAL() */ +template <typename T> +T&& inline_check_non_fatal(T&& val, const char* file, int line, const char* func, const char* assertion) +{ + if (!(val)) { + throw NonFatalCheckError( + format_internal_error(assertion, file, line, func, PACKAGE_BUGREPORT)); + } + return std::forward<T>(val); +} + /** - * Throw a NonFatalCheckError when the condition evaluates to false + * Identity function. Throw a NonFatalCheckError when the condition evaluates to false * * This should only be used * - where the condition is assumed to be true, not for error handling or validating user input @@ -29,32 +44,34 @@ class NonFatalCheckError : public std::runtime_error * asserts or recoverable logic errors. A NonFatalCheckError in RPC code is caught and passed as a string to the RPC * caller, which can then report the issue to the developers. */ -#define CHECK_NONFATAL(condition) \ - do { \ - if (!(condition)) { \ - throw NonFatalCheckError( \ - strprintf("Internal bug detected: '%s'\n" \ - "%s:%d (%s)\n" \ - "You may report this issue here: %s\n", \ - (#condition), \ - __FILE__, __LINE__, __func__, \ - PACKAGE_BUGREPORT)); \ - } \ - } while (false) +#define CHECK_NONFATAL(condition) \ + inline_check_non_fatal(condition, __FILE__, __LINE__, __func__, #condition) #if defined(NDEBUG) #error "Cannot compile without assertions!" #endif /** Helper for Assert() */ -template <typename T> -T get_pure_r_value(T&& val) +void assertion_fail(const char* file, int line, const char* func, const char* assertion); + +/** Helper for Assert()/Assume() */ +template <bool IS_ASSERT, typename T> +T&& inline_assertion_check(T&& val, [[maybe_unused]] const char* file, [[maybe_unused]] int line, [[maybe_unused]] const char* func, [[maybe_unused]] const char* assertion) { + if constexpr (IS_ASSERT +#ifdef ABORT_ON_FAILED_ASSUME + || true +#endif + ) { + if (!val) { + assertion_fail(file, line, func, assertion); + } + } return std::forward<T>(val); } /** Identity function. Abort if the value compares equal to zero */ -#define Assert(val) ([&]() -> decltype(get_pure_r_value(val)) { auto&& check = (val); assert(#val && check); return std::forward<decltype(get_pure_r_value(val))>(check); }()) +#define Assert(val) inline_assertion_check<true>(val, __FILE__, __LINE__, __func__, #val) /** * Assume is the identity function. @@ -66,10 +83,15 @@ T get_pure_r_value(T&& val) * - For non-fatal errors in interactive sessions (e.g. RPC or command line * interfaces), CHECK_NONFATAL() might be more appropriate. */ -#ifdef ABORT_ON_FAILED_ASSUME -#define Assume(val) Assert(val) -#else -#define Assume(val) ([&]() -> decltype(get_pure_r_value(val)) { auto&& check = (val); return std::forward<decltype(get_pure_r_value(val))>(check); }()) -#endif +#define Assume(val) inline_assertion_check<false>(val, __FILE__, __LINE__, __func__, #val) + +/** + * NONFATAL_UNREACHABLE() is a macro that is used to mark unreachable code. It throws a NonFatalCheckError. + * This is used to mark code that is not yet implemented or is not yet reachable. + */ +#define NONFATAL_UNREACHABLE() \ + throw NonFatalCheckError( \ + format_internal_error("Unreachable code reached (non-fatal)", \ + __FILE__, __LINE__, __func__, PACKAGE_BUGREPORT)) #endif // BITCOIN_UTIL_CHECK_H diff --git a/src/util/epochguard.h b/src/util/epochguard.h index 0fec7d2624..7f6477fb3b 100644 --- a/src/util/epochguard.h +++ b/src/util/epochguard.h @@ -7,6 +7,7 @@ #define BITCOIN_UTIL_EPOCHGUARD_H #include <threadsafety.h> +#include <util/macros.h> #include <cassert> @@ -96,6 +97,6 @@ public: } }; -#define WITH_FRESH_EPOCH(epoch) const Epoch::Guard PASTE2(epoch_guard_, __COUNTER__)(epoch) +#define WITH_FRESH_EPOCH(epoch) const Epoch::Guard UNIQUE_NAME(epoch_guard_)(epoch) #endif // BITCOIN_UTIL_EPOCHGUARD_H diff --git a/src/util/getuniquepath.cpp b/src/util/getuniquepath.cpp index 6776e7785b..1d8e511c83 100644 --- a/src/util/getuniquepath.cpp +++ b/src/util/getuniquepath.cpp @@ -9,6 +9,6 @@ fs::path GetUniquePath(const fs::path& base) { FastRandomContext rnd; - fs::path tmpFile = base / HexStr(rnd.randbytes(8)); + fs::path tmpFile = base / fs::u8path(HexStr(rnd.randbytes(8))); return tmpFile; }
\ No newline at end of file diff --git a/src/util/macros.h b/src/util/macros.h index c9740c8e82..bf6ba665dc 100644 --- a/src/util/macros.h +++ b/src/util/macros.h @@ -8,6 +8,8 @@ #define PASTE(x, y) x ## y #define PASTE2(x, y) PASTE(x, y) +#define UNIQUE_NAME(name) PASTE2(name, __COUNTER__) + /** * Converts the parameter X to a string after macro replacement on X has been performed. * Don't merge these into one macro! diff --git a/src/util/message.cpp b/src/util/message.cpp index 2c7f0406f0..f58876f915 100644 --- a/src/util/message.cpp +++ b/src/util/message.cpp @@ -35,14 +35,13 @@ MessageVerificationResult MessageVerify( return MessageVerificationResult::ERR_ADDRESS_NO_KEY; } - bool invalid = false; - std::vector<unsigned char> signature_bytes = DecodeBase64(signature.c_str(), &invalid); - if (invalid) { + auto signature_bytes = DecodeBase64(signature); + if (!signature_bytes) { return MessageVerificationResult::ERR_MALFORMED_SIGNATURE; } CPubKey pubkey; - if (!pubkey.RecoverCompact(MessageHash(message), signature_bytes)) { + if (!pubkey.RecoverCompact(MessageHash(message), *signature_bytes)) { return MessageVerificationResult::ERR_PUBKEY_NOT_RECOVERED; } diff --git a/src/util/moneystr.cpp b/src/util/moneystr.cpp index 2cd7a426f8..8c4bc6e6f4 100644 --- a/src/util/moneystr.cpp +++ b/src/util/moneystr.cpp @@ -40,7 +40,7 @@ std::string FormatMoney(const CAmount n) std::optional<CAmount> ParseMoney(const std::string& money_string) { - if (!ValidAsCString(money_string)) { + if (!ContainsNoNUL(money_string)) { return std::nullopt; } const std::string str = TrimString(money_string); diff --git a/src/util/readwritefile.cpp b/src/util/readwritefile.cpp index a45c41d367..628e6a3980 100644 --- a/src/util/readwritefile.cpp +++ b/src/util/readwritefile.cpp @@ -18,7 +18,7 @@ std::pair<bool,std::string> ReadBinaryFile(const fs::path &filename, size_t maxs std::string retval; char buffer[128]; do { - const size_t n = fread(buffer, 1, sizeof(buffer), f); + const size_t n = fread(buffer, 1, std::min(sizeof(buffer), maxsize - retval.size()), f); // Check for reading errors so we don't return any data if we couldn't // read the entire file (or up to maxsize) if (ferror(f)) { @@ -26,7 +26,7 @@ std::pair<bool,std::string> ReadBinaryFile(const fs::path &filename, size_t maxs return std::make_pair(false,""); } retval.append(buffer, buffer+n); - } while (!feof(f) && retval.size() <= maxsize); + } while (!feof(f) && retval.size() < maxsize); fclose(f); return std::make_pair(true,retval); } diff --git a/src/util/sock.cpp b/src/util/sock.cpp index 2029d70a37..3579af4458 100644 --- a/src/util/sock.cpp +++ b/src/util/sock.cpp @@ -7,6 +7,7 @@ #include <threadinterrupt.h> #include <tinyformat.h> #include <util/sock.h> +#include <util/syserror.h> #include <util/system.h> #include <util/time.h> @@ -105,6 +106,11 @@ int Sock::GetSockOpt(int level, int opt_name, void* opt_val, socklen_t* opt_len) return getsockopt(m_socket, level, opt_name, static_cast<char*>(opt_val), opt_len); } +int Sock::SetSockOpt(int level, int opt_name, const void* opt_val, socklen_t opt_len) const +{ + return setsockopt(m_socket, level, opt_name, static_cast<const char*>(opt_val), opt_len); +} + bool Sock::Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred) const { #ifdef USE_POLL @@ -339,19 +345,8 @@ std::string NetworkErrorString(int err) #else std::string NetworkErrorString(int err) { - char buf[256]; - buf[0] = 0; - /* Too bad there are two incompatible implementations of the - * thread-safe strerror. */ - const char *s; -#ifdef STRERROR_R_CHAR_P /* GNU variant can return a pointer outside the passed buffer */ - s = strerror_r(err, buf, sizeof(buf)); -#else /* POSIX variant always returns message in buffer */ - s = buf; - if (strerror_r(err, buf, sizeof(buf))) - buf[0] = 0; -#endif - return strprintf("%s (%d)", s, err); + // On BSD sockets implementations, NetworkErrorString is the same as SysErrorString. + return SysErrorString(err); } #endif diff --git a/src/util/sock.h b/src/util/sock.h index 7510482857..dd2913a66c 100644 --- a/src/util/sock.h +++ b/src/util/sock.h @@ -115,6 +115,16 @@ public: void* opt_val, socklen_t* opt_len) const; + /** + * setsockopt(2) wrapper. Equivalent to + * `setsockopt(this->Get(), level, opt_name, opt_val, opt_len)`. Code that uses this + * wrapper can be unit tested if this method is overridden by a mock Sock implementation. + */ + [[nodiscard]] virtual int SetSockOpt(int level, + int opt_name, + const void* opt_val, + socklen_t opt_len) const; + using Event = uint8_t; /** diff --git a/src/util/spanparsing.cpp b/src/util/spanparsing.cpp index 50f6aee419..8614bd1176 100644 --- a/src/util/spanparsing.cpp +++ b/src/util/spanparsing.cpp @@ -48,20 +48,4 @@ Span<const char> Expr(Span<const char>& sp) return ret; } -std::vector<Span<const char>> Split(const Span<const char>& sp, char sep) -{ - std::vector<Span<const char>> ret; - auto it = sp.begin(); - auto start = it; - while (it != sp.end()) { - if (*it == sep) { - ret.emplace_back(start, it); - start = it + 1; - } - ++it; - } - ret.emplace_back(start, it); - return ret; -} - } // namespace spanparsing diff --git a/src/util/spanparsing.h b/src/util/spanparsing.h index fa2e698e6d..51795271de 100644 --- a/src/util/spanparsing.h +++ b/src/util/spanparsing.h @@ -8,6 +8,7 @@ #include <span.h> #include <string> +#include <string_view> #include <vector> namespace spanparsing { @@ -36,6 +37,30 @@ bool Func(const std::string& str, Span<const char>& sp); */ Span<const char> Expr(Span<const char>& sp); +/** Split a string on any char found in separators, returning a vector. + * + * If sep does not occur in sp, a singleton with the entirety of sp is returned. + * + * Note that this function does not care about braces, so splitting + * "foo(bar(1),2),3) on ',' will return {"foo(bar(1)", "2)", "3)"}. + */ +template <typename T = Span<const char>> +std::vector<T> Split(const Span<const char>& sp, std::string_view separators) +{ + std::vector<T> ret; + auto it = sp.begin(); + auto start = it; + while (it != sp.end()) { + if (separators.find(*it) != std::string::npos) { + ret.emplace_back(start, it); + start = it + 1; + } + ++it; + } + ret.emplace_back(start, it); + return ret; +} + /** Split a string on every instance of sep, returning a vector. * * If sep does not occur in sp, a singleton with the entirety of sp is returned. @@ -43,7 +68,11 @@ Span<const char> Expr(Span<const char>& sp); * Note that this function does not care about braces, so splitting * "foo(bar(1),2),3) on ',' will return {"foo(bar(1)", "2)", "3)"}. */ -std::vector<Span<const char>> Split(const Span<const char>& sp, char sep); +template <typename T = Span<const char>> +std::vector<T> Split(const Span<const char>& sp, char sep) +{ + return Split<T>(sp, std::string_view{&sep, 1}); +} } // namespace spanparsing diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index 940fa90da2..bcedd4f517 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -9,6 +9,7 @@ #include <tinyformat.h> #include <algorithm> +#include <array> #include <cstdlib> #include <cstring> #include <limits> @@ -24,15 +25,15 @@ static const std::string SAFE_CHARS[] = CHARS_ALPHA_NUM + "!*'();:@&=+$,/?#[]-_.~%", // SAFE_CHARS_URI }; -std::string SanitizeString(const std::string& str, int rule) +std::string SanitizeString(std::string_view str, int rule) { - std::string strResult; - for (std::string::size_type i = 0; i < str.size(); i++) - { - if (SAFE_CHARS[rule].find(str[i]) != std::string::npos) - strResult.push_back(str[i]); + std::string result; + for (char c : str) { + if (SAFE_CHARS[rule].find(c) != std::string::npos) { + result.push_back(c); + } } - return strResult; + return result; } const signed char p_util_hexdigit[256] = @@ -58,56 +59,43 @@ signed char HexDigit(char c) return p_util_hexdigit[(unsigned char)c]; } -bool IsHex(const std::string& str) +bool IsHex(std::string_view str) { - for(std::string::const_iterator it(str.begin()); it != str.end(); ++it) - { - if (HexDigit(*it) < 0) - return false; + for (char c : str) { + if (HexDigit(c) < 0) return false; } return (str.size() > 0) && (str.size()%2 == 0); } -bool IsHexNumber(const std::string& str) +bool IsHexNumber(std::string_view str) { - size_t starting_location = 0; - if (str.size() > 2 && *str.begin() == '0' && *(str.begin()+1) == 'x') { - starting_location = 2; - } - for (const char c : str.substr(starting_location)) { + if (str.substr(0, 2) == "0x") str.remove_prefix(2); + for (char c : str) { if (HexDigit(c) < 0) return false; } // Return false for empty string or "0x". - return (str.size() > starting_location); + return str.size() > 0; } -std::vector<unsigned char> ParseHex(const char* psz) +std::vector<unsigned char> ParseHex(std::string_view str) { // convert hex dump to vector std::vector<unsigned char> vch; - while (true) - { - while (IsSpace(*psz)) - psz++; - signed char c = HexDigit(*psz++); - if (c == (signed char)-1) - break; - auto n{uint8_t(c << 4)}; - c = HexDigit(*psz++); - if (c == (signed char)-1) - break; - n |= c; - vch.push_back(n); + auto it = str.begin(); + while (it != str.end() && it + 1 != str.end()) { + if (IsSpace(*it)) { + ++it; + continue; + } + auto c1 = HexDigit(*(it++)); + auto c2 = HexDigit(*(it++)); + if (c1 < 0 || c2 < 0) break; + vch.push_back(uint8_t(c1 << 4) | c2); } return vch; } -std::vector<unsigned char> ParseHex(const std::string& str) -{ - return ParseHex(str.c_str()); -} - -void SplitHostPort(std::string in, uint16_t& portOut, std::string& hostOut) +void SplitHostPort(std::string_view in, uint16_t& portOut, std::string& hostOut) { size_t colon = in.find_last_of(':'); // if a : is found, and it either follows a [...], or no other : is in the string, treat it as port separator @@ -139,7 +127,7 @@ std::string EncodeBase64(Span<const unsigned char> input) return str; } -std::vector<unsigned char> DecodeBase64(const char* p, bool* pf_invalid) +std::optional<std::vector<unsigned char>> DecodeBase64(std::string_view str) { static const int8_t decode64_table[256]{ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -157,46 +145,23 @@ std::vector<unsigned char> DecodeBase64(const char* p, bool* pf_invalid) -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }; - const char* e = p; - std::vector<uint8_t> val; - val.reserve(strlen(p)); - while (*p != 0) { - int x = decode64_table[(unsigned char)*p]; - if (x == -1) break; - val.push_back(uint8_t(x)); - ++p; - } + if (str.size() % 4 != 0) return {}; + /* One or two = characters at the end are permitted. */ + if (str.size() >= 1 && str.back() == '=') str.remove_suffix(1); + if (str.size() >= 1 && str.back() == '=') str.remove_suffix(1); std::vector<unsigned char> ret; - ret.reserve((val.size() * 3) / 4); - bool valid = ConvertBits<6, 8, false>([&](unsigned char c) { ret.push_back(c); }, val.begin(), val.end()); - - const char* q = p; - while (valid && *p != 0) { - if (*p != '=') { - valid = false; - break; - } - ++p; - } - valid = valid && (p - e) % 4 == 0 && p - q < 4; - if (pf_invalid) *pf_invalid = !valid; + ret.reserve((str.size() * 3) / 4); + bool valid = ConvertBits<6, 8, false>( + [&](unsigned char c) { ret.push_back(c); }, + str.begin(), str.end(), + [](char c) { return decode64_table[uint8_t(c)]; } + ); + if (!valid) return {}; return ret; } -std::string DecodeBase64(const std::string& str, bool* pf_invalid) -{ - if (!ValidAsCString(str)) { - if (pf_invalid) { - *pf_invalid = true; - } - return {}; - } - std::vector<unsigned char> vchRet = DecodeBase64(str.c_str(), pf_invalid); - return std::string((const char*)vchRet.data(), vchRet.size()); -} - std::string EncodeBase32(Span<const unsigned char> input, bool pad) { static const char *pbase32 = "abcdefghijklmnopqrstuvwxyz234567"; @@ -212,12 +177,12 @@ std::string EncodeBase32(Span<const unsigned char> input, bool pad) return str; } -std::string EncodeBase32(const std::string& str, bool pad) +std::string EncodeBase32(std::string_view str, bool pad) { return EncodeBase32(MakeUCharSpan(str), pad); } -std::vector<unsigned char> DecodeBase32(const char* p, bool* pf_invalid) +std::optional<std::vector<unsigned char>> DecodeBase32(std::string_view str) { static const int8_t decode32_table[256]{ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -235,49 +200,29 @@ std::vector<unsigned char> DecodeBase32(const char* p, bool* pf_invalid) -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }; - const char* e = p; - std::vector<uint8_t> val; - val.reserve(strlen(p)); - while (*p != 0) { - int x = decode32_table[(unsigned char)*p]; - if (x == -1) break; - val.push_back(uint8_t(x)); - ++p; - } + if (str.size() % 8 != 0) return {}; + /* 1, 3, 4, or 6 padding '=' suffix characters are permitted. */ + if (str.size() >= 1 && str.back() == '=') str.remove_suffix(1); + if (str.size() >= 2 && str.substr(str.size() - 2) == "==") str.remove_suffix(2); + if (str.size() >= 1 && str.back() == '=') str.remove_suffix(1); + if (str.size() >= 2 && str.substr(str.size() - 2) == "==") str.remove_suffix(2); std::vector<unsigned char> ret; - ret.reserve((val.size() * 5) / 8); - bool valid = ConvertBits<5, 8, false>([&](unsigned char c) { ret.push_back(c); }, val.begin(), val.end()); - - const char* q = p; - while (valid && *p != 0) { - if (*p != '=') { - valid = false; - break; - } - ++p; - } - valid = valid && (p - e) % 8 == 0 && p - q < 8; - if (pf_invalid) *pf_invalid = !valid; + ret.reserve((str.size() * 5) / 8); + bool valid = ConvertBits<5, 8, false>( + [&](unsigned char c) { ret.push_back(c); }, + str.begin(), str.end(), + [](char c) { return decode32_table[uint8_t(c)]; } + ); - return ret; -} + if (!valid) return {}; -std::string DecodeBase32(const std::string& str, bool* pf_invalid) -{ - if (!ValidAsCString(str)) { - if (pf_invalid) { - *pf_invalid = true; - } - return {}; - } - std::vector<unsigned char> vchRet = DecodeBase32(str.c_str(), pf_invalid); - return std::string((const char*)vchRet.data(), vchRet.size()); + return ret; } namespace { template <typename T> -bool ParseIntegral(const std::string& str, T* out) +bool ParseIntegral(std::string_view str, T* out) { static_assert(std::is_integral<T>::value); // Replicate the exact behavior of strtol/strtoll/strtoul/strtoull when @@ -296,37 +241,37 @@ bool ParseIntegral(const std::string& str, T* out) } }; // namespace -bool ParseInt32(const std::string& str, int32_t* out) +bool ParseInt32(std::string_view str, int32_t* out) { return ParseIntegral<int32_t>(str, out); } -bool ParseInt64(const std::string& str, int64_t* out) +bool ParseInt64(std::string_view str, int64_t* out) { return ParseIntegral<int64_t>(str, out); } -bool ParseUInt8(const std::string& str, uint8_t* out) +bool ParseUInt8(std::string_view str, uint8_t* out) { return ParseIntegral<uint8_t>(str, out); } -bool ParseUInt16(const std::string& str, uint16_t* out) +bool ParseUInt16(std::string_view str, uint16_t* out) { return ParseIntegral<uint16_t>(str, out); } -bool ParseUInt32(const std::string& str, uint32_t* out) +bool ParseUInt32(std::string_view str, uint32_t* out) { return ParseIntegral<uint32_t>(str, out); } -bool ParseUInt64(const std::string& str, uint64_t* out) +bool ParseUInt64(std::string_view str, uint64_t* out) { return ParseIntegral<uint64_t>(str, out); } -std::string FormatParagraph(const std::string& in, size_t width, size_t indent) +std::string FormatParagraph(std::string_view in, size_t width, size_t indent) { assert(width >= indent); std::stringstream out; @@ -395,7 +340,7 @@ static inline bool ProcessMantissaDigit(char ch, int64_t &mantissa, int &mantiss return true; } -bool ParseFixedPoint(const std::string &val, int decimals, int64_t *amount_out) +bool ParseFixedPoint(std::string_view val, int decimals, int64_t *amount_out) { int64_t mantissa = 0; int64_t exponent = 0; @@ -487,14 +432,14 @@ bool ParseFixedPoint(const std::string &val, int decimals, int64_t *amount_out) return true; } -std::string ToLower(const std::string& str) +std::string ToLower(std::string_view str) { std::string r; for (auto ch : str) r += ToLower(ch); return r; } -std::string ToUpper(const std::string& str) +std::string ToUpper(std::string_view str) { std::string r; for (auto ch : str) r += ToUpper(ch); @@ -508,21 +453,41 @@ std::string Capitalize(std::string str) return str; } +namespace { + +using ByteAsHex = std::array<char, 2>; + +constexpr std::array<ByteAsHex, 256> CreateByteToHexMap() +{ + constexpr char hexmap[16] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + + std::array<ByteAsHex, 256> byte_to_hex{}; + for (size_t i = 0; i < byte_to_hex.size(); ++i) { + byte_to_hex[i][0] = hexmap[i >> 4]; + byte_to_hex[i][1] = hexmap[i & 15]; + } + return byte_to_hex; +} + +} // namespace + std::string HexStr(const Span<const uint8_t> s) { std::string rv(s.size() * 2, '\0'); - static constexpr char hexmap[16] = { '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; - auto it = rv.begin(); + static constexpr auto byte_to_hex = CreateByteToHexMap(); + static_assert(sizeof(byte_to_hex) == 512); + + char* it = rv.data(); for (uint8_t v : s) { - *it++ = hexmap[v >> 4]; - *it++ = hexmap[v & 15]; + std::memcpy(it, byte_to_hex[v].data(), 2); + it += 2; } - assert(it == rv.end()); + + assert(it == rv.data() + rv.size()); return rv; } -std::optional<uint64_t> ParseByteUnits(const std::string& str, ByteUnit default_multiplier) +std::optional<uint64_t> ParseByteUnits(std::string_view str, ByteUnit default_multiplier) { if (str.empty()) { return std::nullopt; diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 1f83fa3ffa..ebb6d88952 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -54,24 +54,21 @@ enum class ByteUnit : uint64_t { * @param[in] rule The set of safe chars to choose (default: least restrictive) * @return A new string without unsafe chars */ -std::string SanitizeString(const std::string& str, int rule = SAFE_CHARS_DEFAULT); -std::vector<unsigned char> ParseHex(const char* psz); -std::vector<unsigned char> ParseHex(const std::string& str); +std::string SanitizeString(std::string_view str, int rule = SAFE_CHARS_DEFAULT); +std::vector<unsigned char> ParseHex(std::string_view str); signed char HexDigit(char c); /* Returns true if each character in str is a hex character, and has an even * number of hex digits.*/ -bool IsHex(const std::string& str); +bool IsHex(std::string_view str); /** * Return true if the string is a hex number, optionally prefixed with "0x" */ -bool IsHexNumber(const std::string& str); -std::vector<unsigned char> DecodeBase64(const char* p, bool* pf_invalid = nullptr); -std::string DecodeBase64(const std::string& str, bool* pf_invalid = nullptr); +bool IsHexNumber(std::string_view str); +std::optional<std::vector<unsigned char>> DecodeBase64(std::string_view str); std::string EncodeBase64(Span<const unsigned char> input); inline std::string EncodeBase64(Span<const std::byte> input) { return EncodeBase64(MakeUCharSpan(input)); } -inline std::string EncodeBase64(const std::string& str) { return EncodeBase64(MakeUCharSpan(str)); } -std::vector<unsigned char> DecodeBase32(const char* p, bool* pf_invalid = nullptr); -std::string DecodeBase32(const std::string& str, bool* pf_invalid = nullptr); +inline std::string EncodeBase64(std::string_view str) { return EncodeBase64(MakeUCharSpan(str)); } +std::optional<std::vector<unsigned char>> DecodeBase32(std::string_view str); /** * Base32 encode. @@ -85,9 +82,9 @@ std::string EncodeBase32(Span<const unsigned char> input, bool pad = true); * If `pad` is true, then the output will be padded with '=' so that its length * is a multiple of 8. */ -std::string EncodeBase32(const std::string& str, bool pad = true); +std::string EncodeBase32(std::string_view str, bool pad = true); -void SplitHostPort(std::string in, uint16_t& portOut, std::string& hostOut); +void SplitHostPort(std::string_view in, uint16_t& portOut, std::string& hostOut); // LocaleIndependentAtoi is provided for backwards compatibility reasons. // @@ -101,12 +98,12 @@ void SplitHostPort(std::string in, uint16_t& portOut, std::string& hostOut); // undefined behavior, while this function returns the maximum or minimum // values, respectively. template <typename T> -T LocaleIndependentAtoi(const std::string& str) +T LocaleIndependentAtoi(std::string_view str) { static_assert(std::is_integral<T>::value); T result; // Emulate atoi(...) handling of white space and leading +/-. - std::string s = TrimString(str); + std::string_view s = TrimStringView(str); if (!s.empty() && s[0] == '+') { if (s.length() >= 2 && s[1] == '-') { return 0; @@ -162,7 +159,7 @@ constexpr inline bool IsSpace(char c) noexcept { * parsed value is not in the range representable by the type T. */ template <typename T> -std::optional<T> ToIntegral(const std::string& str) +std::optional<T> ToIntegral(std::string_view str) { static_assert(std::is_integral<T>::value); T result; @@ -178,42 +175,42 @@ std::optional<T> ToIntegral(const std::string& str) * @returns true if the entire string could be parsed as valid integer, * false if not the entire string could be parsed or when overflow or underflow occurred. */ -[[nodiscard]] bool ParseInt32(const std::string& str, int32_t *out); +[[nodiscard]] bool ParseInt32(std::string_view str, int32_t *out); /** * Convert string to signed 64-bit integer with strict parse error feedback. * @returns true if the entire string could be parsed as valid integer, * false if not the entire string could be parsed or when overflow or underflow occurred. */ -[[nodiscard]] bool ParseInt64(const std::string& str, int64_t *out); +[[nodiscard]] bool ParseInt64(std::string_view str, int64_t *out); /** * Convert decimal string to unsigned 8-bit integer with strict parse error feedback. * @returns true if the entire string could be parsed as valid integer, * false if not the entire string could be parsed or when overflow or underflow occurred. */ -[[nodiscard]] bool ParseUInt8(const std::string& str, uint8_t *out); +[[nodiscard]] bool ParseUInt8(std::string_view str, uint8_t *out); /** * Convert decimal string to unsigned 16-bit integer with strict parse error feedback. * @returns true if the entire string could be parsed as valid integer, * false if the entire string could not be parsed or if overflow or underflow occurred. */ -[[nodiscard]] bool ParseUInt16(const std::string& str, uint16_t* out); +[[nodiscard]] bool ParseUInt16(std::string_view str, uint16_t* out); /** * Convert decimal string to unsigned 32-bit integer with strict parse error feedback. * @returns true if the entire string could be parsed as valid integer, * false if not the entire string could be parsed or when overflow or underflow occurred. */ -[[nodiscard]] bool ParseUInt32(const std::string& str, uint32_t *out); +[[nodiscard]] bool ParseUInt32(std::string_view str, uint32_t *out); /** * Convert decimal string to unsigned 64-bit integer with strict parse error feedback. * @returns true if the entire string could be parsed as valid integer, * false if not the entire string could be parsed or when overflow or underflow occurred. */ -[[nodiscard]] bool ParseUInt64(const std::string& str, uint64_t *out); +[[nodiscard]] bool ParseUInt64(std::string_view str, uint64_t *out); /** * Convert a span of bytes to a lower-case hexadecimal string. @@ -226,7 +223,7 @@ inline std::string HexStr(const Span<const std::byte> s) { return HexStr(MakeUCh * Format a paragraph of text to a fixed width, adding spaces for * indentation to any added line. */ -std::string FormatParagraph(const std::string& in, size_t width = 79, size_t indent = 0); +std::string FormatParagraph(std::string_view in, size_t width = 79, size_t indent = 0); /** * Timing-attack-resistant comparison. @@ -248,17 +245,28 @@ bool TimingResistantEqual(const T& a, const T& b) * @returns true on success, false on error. * @note The result must be in the range (-10^18,10^18), otherwise an overflow error will trigger. */ -[[nodiscard]] bool ParseFixedPoint(const std::string &val, int decimals, int64_t *amount_out); +[[nodiscard]] bool ParseFixedPoint(std::string_view, int decimals, int64_t *amount_out); + +namespace { +/** Helper class for the default infn argument to ConvertBits (just returns the input). */ +struct IntIdentity +{ + [[maybe_unused]] int operator()(int x) const { return x; } +}; + +} // namespace /** Convert from one power-of-2 number base to another. */ -template<int frombits, int tobits, bool pad, typename O, typename I> -bool ConvertBits(const O& outfn, I it, I end) { +template<int frombits, int tobits, bool pad, typename O, typename It, typename I = IntIdentity> +bool ConvertBits(O outfn, It it, It end, I infn = {}) { size_t acc = 0; size_t bits = 0; constexpr size_t maxv = (1 << tobits) - 1; constexpr size_t max_acc = (1 << (frombits + tobits - 1)) - 1; while (it != end) { - acc = ((acc << frombits) | *it) & max_acc; + int v = infn(*it); + if (v < 0) return false; + acc = ((acc << frombits) | v) & max_acc; bits += frombits; while (bits >= tobits) { bits -= tobits; @@ -298,7 +306,7 @@ constexpr char ToLower(char c) * @param[in] str the string to convert to lowercase. * @returns lowercased equivalent of str */ -std::string ToLower(const std::string& str); +std::string ToLower(std::string_view str); /** * Converts the given character to its uppercase equivalent. @@ -324,7 +332,7 @@ constexpr char ToUpper(char c) * @param[in] str the string to convert to uppercase. * @returns UPPERCASED EQUIVALENT OF str */ -std::string ToUpper(const std::string& str); +std::string ToUpper(std::string_view str); /** * Capitalizes the first character of the given string. @@ -348,6 +356,6 @@ std::string Capitalize(std::string str); * @returns optional uint64_t bytes from str or nullopt * if ToIntegral is false, str is empty, trailing whitespace or overflow */ -std::optional<uint64_t> ParseByteUnits(const std::string& str, ByteUnit default_multiplier); +std::optional<uint64_t> ParseByteUnits(std::string_view str, ByteUnit default_multiplier); #endif // BITCOIN_UTIL_STRENCODINGS_H diff --git a/src/util/string.cpp b/src/util/string.cpp index 8ea3a1afc6..d05222e8b8 100644 --- a/src/util/string.cpp +++ b/src/util/string.cpp @@ -3,3 +3,10 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <util/string.h> + +#include <boost/algorithm/string/replace.hpp> + +void ReplaceAll(std::string& in_out, std::string_view search, std::string_view substitute) +{ + boost::replace_all(in_out, search, substitute); +} diff --git a/src/util/string.h b/src/util/string.h index a3b8df8d78..df20e34ae9 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -5,27 +5,46 @@ #ifndef BITCOIN_UTIL_STRING_H #define BITCOIN_UTIL_STRING_H -#include <attributes.h> +#include <util/spanparsing.h> #include <algorithm> #include <array> +#include <cstdint> #include <cstring> #include <locale> #include <sstream> #include <string> +#include <string_view> #include <vector> -[[nodiscard]] inline std::string TrimString(const std::string& str, const std::string& pattern = " \f\n\r\t\v") +void ReplaceAll(std::string& in_out, std::string_view search, std::string_view substitute); + +[[nodiscard]] inline std::vector<std::string> SplitString(std::string_view str, char sep) +{ + return spanparsing::Split<std::string>(str, sep); +} + +[[nodiscard]] inline std::vector<std::string> SplitString(std::string_view str, std::string_view separators) +{ + return spanparsing::Split<std::string>(str, separators); +} + +[[nodiscard]] inline std::string_view TrimStringView(std::string_view str, std::string_view pattern = " \f\n\r\t\v") { std::string::size_type front = str.find_first_not_of(pattern); if (front == std::string::npos) { - return std::string(); + return {}; } std::string::size_type end = str.find_last_not_of(pattern); return str.substr(front, end - front + 1); } -[[nodiscard]] inline std::string RemovePrefix(const std::string& str, const std::string& prefix) +[[nodiscard]] inline std::string TrimString(std::string_view str, std::string_view pattern = " \f\n\r\t\v") +{ + return std::string(TrimStringView(str, pattern)); +} + +[[nodiscard]] inline std::string_view RemovePrefixView(std::string_view str, std::string_view prefix) { if (str.substr(0, prefix.size()) == prefix) { return str.substr(prefix.size()); @@ -33,6 +52,11 @@ return str; } +[[nodiscard]] inline std::string RemovePrefix(std::string_view str, std::string_view prefix) +{ + return std::string(RemovePrefixView(str, prefix)); +} + /** * Join a list of items * @@ -52,14 +76,14 @@ auto Join(const std::vector<T>& list, const BaseType& separator, UnaryOp unary_o return ret; } -template <typename T> -T Join(const std::vector<T>& list, const T& separator) +template <typename T, typename T2> +T Join(const std::vector<T>& list, const T2& separator) { return Join(list, separator, [](const T& i) { return i; }); } // Explicit overload needed for c_str arguments, which would otherwise cause a substitution failure in the template above. -inline std::string Join(const std::vector<std::string>& list, const std::string& separator) +inline std::string Join(const std::vector<std::string>& list, std::string_view separator) { return Join<std::string>(list, separator); } @@ -75,9 +99,12 @@ inline std::string MakeUnorderedList(const std::vector<std::string>& items) /** * Check if a string does not contain any embedded NUL (\0) characters */ -[[nodiscard]] inline bool ValidAsCString(const std::string& str) noexcept +[[nodiscard]] inline bool ContainsNoNUL(std::string_view str) noexcept { - return str.size() == strlen(str.c_str()); + for (auto c : str) { + if (c == 0) return false; + } + return true; } /** diff --git a/src/util/syscall_sandbox.cpp b/src/util/syscall_sandbox.cpp index f2a9cf664d..a69f815ce4 100644 --- a/src/util/syscall_sandbox.cpp +++ b/src/util/syscall_sandbox.cpp @@ -821,7 +821,6 @@ bool SetupSyscallSandbox(bool log_syscall_violation_before_terminating) return false; } } - SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION); return true; } diff --git a/src/util/syscall_sandbox.h b/src/util/syscall_sandbox.h index f7a1cbdb55..dc02ce29e9 100644 --- a/src/util/syscall_sandbox.h +++ b/src/util/syscall_sandbox.h @@ -45,9 +45,6 @@ void SetSyscallSandboxPolicy(SyscallSandboxPolicy syscall_policy); #if defined(USE_SYSCALL_SANDBOX) //! Setup and enable the experimental syscall sandbox for the running process. -//! -//! SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION) is called as part of -//! SetupSyscallSandbox(...). [[nodiscard]] bool SetupSyscallSandbox(bool log_syscall_violation_before_terminating); //! Invoke a disallowed syscall. Use for testing purposes. diff --git a/src/util/syserror.cpp b/src/util/syserror.cpp new file mode 100644 index 0000000000..391ddd3560 --- /dev/null +++ b/src/util/syserror.cpp @@ -0,0 +1,34 @@ +// Copyright (c) 2020-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <tinyformat.h> +#include <util/syserror.h> + +#include <cstring> + +std::string SysErrorString(int err) +{ + char buf[1024]; + /* Too bad there are three incompatible implementations of the + * thread-safe strerror. */ + const char *s = nullptr; +#ifdef WIN32 + if (strerror_s(buf, sizeof(buf), err) == 0) s = buf; +#else +#ifdef STRERROR_R_CHAR_P /* GNU variant can return a pointer outside the passed buffer */ + s = strerror_r(err, buf, sizeof(buf)); +#else /* POSIX variant always returns message in buffer */ + if (strerror_r(err, buf, sizeof(buf)) == 0) s = buf; +#endif +#endif + if (s != nullptr) { + return strprintf("%s (%d)", s, err); + } else { + return strprintf("Unknown error (%d)", err); + } +} diff --git a/src/util/syserror.h b/src/util/syserror.h new file mode 100644 index 0000000000..a54ba553ee --- /dev/null +++ b/src/util/syserror.h @@ -0,0 +1,16 @@ +// Copyright (c) 2010-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_SYSERROR_H +#define BITCOIN_UTIL_SYSERROR_H + +#include <string> + +/** Return system error string from errno value. Use this instead of + * std::strerror, which is not thread-safe. For network errors use + * NetworkErrorString from sock.h instead. + */ +std::string SysErrorString(int err); + +#endif // BITCOIN_UTIL_SYSERROR_H diff --git a/src/util/system.cpp b/src/util/system.cpp index aa9122106b..44ebf5cb3e 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -6,7 +6,16 @@ #include <util/system.h> #ifdef ENABLE_EXTERNAL_SIGNER +#if defined(__GNUC__) +// Boost 1.78 requires the following workaround. +// See: https://github.com/boostorg/process/issues/235 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wnarrowing" +#endif #include <boost/process.hpp> +#if defined(__GNUC__) +#pragma GCC diagnostic pop +#endif #endif // ENABLE_EXTERNAL_SIGNER #include <chainparamsbase.h> @@ -16,6 +25,7 @@ #include <util/getuniquepath.h> #include <util/strencodings.h> #include <util/string.h> +#include <util/syserror.h> #include <util/translation.h> @@ -66,7 +76,6 @@ #include <malloc.h> #endif -#include <boost/algorithm/string/replace.hpp> #include <univalue.h> #include <fstream> @@ -95,7 +104,7 @@ static Mutex cs_dir_locks; */ static std::map<std::string, std::unique_ptr<fsbridge::FileLock>> dir_locks GUARDED_BY(cs_dir_locks); -bool LockDirectory(const fs::path& directory, const std::string lockfile_name, bool probe_only) +bool LockDirectory(const fs::path& directory, const fs::path& lockfile_name, bool probe_only) { LOCK(cs_dir_locks); fs::path pathLockFile = directory / lockfile_name; @@ -119,7 +128,7 @@ bool LockDirectory(const fs::path& directory, const std::string lockfile_name, b return true; } -void UnlockDirectory(const fs::path& directory, const std::string& lockfile_name) +void UnlockDirectory(const fs::path& directory, const fs::path& lockfile_name) { LOCK(cs_dir_locks); dir_locks.erase(fs::PathToString(directory / lockfile_name)); @@ -387,9 +396,12 @@ std::optional<unsigned int> ArgsManager::GetArgFlags(const std::string& name) co return std::nullopt; } -fs::path ArgsManager::GetPathArg(std::string pathlike_arg) const +fs::path ArgsManager::GetPathArg(std::string arg, const fs::path& default_value) const { - auto result = fs::PathFromString(GetArg(pathlike_arg, "")).lexically_normal(); + if (IsArgNegated(arg)) return fs::path{}; + std::string path_str = GetArg(arg, ""); + if (path_str.empty()) return default_value; + fs::path result = fs::PathFromString(path_str).lexically_normal(); // Remove trailing slash, if present. return result.has_filename() ? result : result.parent_path(); } @@ -516,12 +528,12 @@ bool ArgsManager::InitSettings(std::string& error) bool ArgsManager::GetSettingsPath(fs::path* filepath, bool temp) const { - if (IsArgNegated("-settings")) { + fs::path settings = GetPathArg("-settings", fs::path{BITCOIN_SETTINGS_FILENAME}); + if (settings.empty()) { return false; } if (filepath) { - std::string settings = GetArg("-settings", BITCOIN_SETTINGS_FILENAME); - *filepath = fsbridge::AbsPathJoin(GetDataDirNet(), fs::PathFromString(temp ? settings + ".tmp" : settings)); + *filepath = fsbridge::AbsPathJoin(GetDataDirNet(), temp ? settings + ".tmp" : settings); } return true; } @@ -588,7 +600,7 @@ bool ArgsManager::IsArgNegated(const std::string& strArg) const std::string ArgsManager::GetArg(const std::string& strArg, const std::string& strDefault) const { const util::SettingsValue value = GetSetting(strArg); - return value.isNull() ? strDefault : value.isFalse() ? "0" : value.isTrue() ? "1" : value.get_str(); + return value.isNull() ? strDefault : value.isFalse() ? "0" : value.isTrue() ? "1" : value.isNum() ? value.getValStr() : value.get_str(); } int64_t ArgsManager::GetIntArg(const std::string& strArg, int64_t nDefault) const @@ -841,8 +853,8 @@ static bool GetConfigOptions(std::istream& stream, const std::string& filepath, error = strprintf("parse error on line %i: %s, options in configuration file must be specified without leading -", linenr, str); return false; } else if ((pos = str.find('=')) != std::string::npos) { - std::string name = prefix + TrimString(str.substr(0, pos), pattern); - std::string value = TrimString(str.substr(pos + 1), pattern); + std::string name = prefix + TrimString(std::string_view{str}.substr(0, pos), pattern); + std::string_view value = TrimStringView(std::string_view{str}.substr(pos + 1), pattern); if (used_hash && name.find("rpcpassword") != std::string::npos) { error = strprintf("parse error on line %i, using # in rpcpassword can be ambiguous and should be avoided", linenr); return false; @@ -1240,7 +1252,7 @@ fs::path GetSpecialFolderPath(int nFolder, bool fCreate) std::string ShellEscape(const std::string& arg) { std::string escaped = arg; - boost::replace_all(escaped, "'", "'\"'\"'"); + ReplaceAll(escaped, "'", "'\"'\"'"); return "'" + escaped + "'"; } #endif @@ -1362,7 +1374,7 @@ void ScheduleBatchPriority() const static sched_param param{}; const int rc = pthread_setschedparam(pthread_self(), SCHED_BATCH, ¶m); if (rc != 0) { - LogPrintf("Failed to pthread_setschedparam: %s\n", strerror(rc)); + LogPrintf("Failed to pthread_setschedparam: %s\n", SysErrorString(rc)); } #endif } diff --git a/src/util/system.h b/src/util/system.h index f193c8ac0b..a7f4d16911 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -76,8 +76,8 @@ void AllocateFileRange(FILE *file, unsigned int offset, unsigned int length); */ [[nodiscard]] bool RenameOver(fs::path src, fs::path dest); -bool LockDirectory(const fs::path& directory, const std::string lockfile_name, bool probe_only=false); -void UnlockDirectory(const fs::path& directory, const std::string& lockfile_name); +bool LockDirectory(const fs::path& directory, const fs::path& lockfile_name, bool probe_only=false); +void UnlockDirectory(const fs::path& directory, const fs::path& lockfile_name); bool DirIsWritable(const fs::path& directory); bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes = 0); @@ -271,16 +271,6 @@ protected: std::optional<const Command> GetCommand() const; /** - * Get a normalized path from a specified pathlike argument - * - * It is guaranteed that the returned path has no trailing slashes. - * - * @param pathlike_arg Pathlike argument to get a path from (e.g., "-datadir", "-blocksdir" or "-walletdir") - * @return Normalized path which is get from a specified pathlike argument - */ - fs::path GetPathArg(std::string pathlike_arg) const; - - /** * Get blocks directory path * * @return Blocks path which is network specific @@ -343,6 +333,18 @@ protected: std::string GetArg(const std::string& strArg, const std::string& strDefault) const; /** + * Return path argument or default value + * + * @param arg Argument to get a path from (e.g., "-datadir", "-blocksdir" or "-walletdir") + * @param default_value Optional default value to return instead of the empty path. + * @return normalized path if argument is set, with redundant "." and ".." + * path components and trailing separators removed (see patharg unit test + * for examples or implementation for details). If argument is empty or not + * set, default_value is returned unchanged. + */ + fs::path GetPathArg(std::string arg, const fs::path& default_value = {}) const; + + /** * Return integer argument or default value * * @param strArg Argument to get (e.g. "-foo") diff --git a/src/util/threadnames.cpp b/src/util/threadnames.cpp index 764fffabd7..a5a86d2598 100644 --- a/src/util/threadnames.cpp +++ b/src/util/threadnames.cpp @@ -6,7 +6,9 @@ #include <config/bitcoin-config.h> #endif +#include <string> #include <thread> +#include <utility> #if (defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)) #include <pthread.h> @@ -16,7 +18,7 @@ #include <util/threadnames.h> #ifdef HAVE_SYS_PRCTL_H -#include <sys/prctl.h> // For prctl, PR_SET_NAME, PR_GET_NAME +#include <sys/prctl.h> #endif //! Set the thread's name at the process level. Does not affect the diff --git a/src/util/time.cpp b/src/util/time.cpp index f7712f0dc8..e428430bac 100644 --- a/src/util/time.cpp +++ b/src/util/time.cpp @@ -23,16 +23,6 @@ void UninterruptibleSleep(const std::chrono::microseconds& n) { std::this_thread static std::atomic<int64_t> nMockTime(0); //!< For testing -int64_t GetTime() -{ - int64_t mocktime = nMockTime.load(std::memory_order_relaxed); - if (mocktime) return mocktime; - - time_t now = time(nullptr); - assert(now > 0); - return now; -} - bool ChronoSanityCheck() { // std::chrono::system_clock.time_since_epoch and time_t(0) are not guaranteed @@ -80,11 +70,12 @@ template <typename T> T GetTime() { const std::chrono::seconds mocktime{nMockTime.load(std::memory_order_relaxed)}; - - return std::chrono::duration_cast<T>( + const auto ret{ mocktime.count() ? mocktime : - std::chrono::microseconds{GetTimeMicros()}); + std::chrono::duration_cast<T>(std::chrono::system_clock::now().time_since_epoch())}; + assert(ret > 0s); + return ret; } template std::chrono::seconds GetTime(); template std::chrono::milliseconds GetTime(); @@ -129,6 +120,8 @@ int64_t GetTimeSeconds() return int64_t{GetSystemTime<std::chrono::seconds>().count()}; } +int64_t GetTime() { return GetTime<std::chrono::seconds>().count(); } + std::string FormatISO8601DateTime(int64_t nTime) { struct tm ts; time_t time_val = nTime; diff --git a/src/util/time.h b/src/util/time.h index 9d92b23725..041b8aa6a1 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -14,6 +14,10 @@ using namespace std::chrono_literals; +using SteadySeconds = std::chrono::time_point<std::chrono::steady_clock, std::chrono::seconds>; +using SteadyMilliseconds = std::chrono::time_point<std::chrono::steady_clock, std::chrono::milliseconds>; +using SteadyMicroseconds = std::chrono::time_point<std::chrono::steady_clock, std::chrono::microseconds>; + void UninterruptibleSleep(const std::chrono::microseconds& n); /** @@ -67,6 +71,15 @@ std::chrono::seconds GetMockTime(); /** Return system time (or mocked time, if set) */ template <typename T> T GetTime(); +/** + * Return the current time point cast to the given precicion. Only use this + * when an exact precicion is needed, otherwise use T::clock::now() directly. + */ +template <typename T> +T Now() +{ + return std::chrono::time_point_cast<typename T::duration>(T::clock::now()); +} /** * ISO 8601 formatting is preferred. Use the FormatISO8601{DateTime,Date} diff --git a/src/util/tokenpipe.cpp b/src/util/tokenpipe.cpp index 4c091cd2e6..49456814e2 100644 --- a/src/util/tokenpipe.cpp +++ b/src/util/tokenpipe.cpp @@ -3,7 +3,9 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <util/tokenpipe.h> +#if defined(HAVE_CONFIG_H) #include <config/bitcoin-config.h> +#endif #ifndef WIN32 diff --git a/src/validation.cpp b/src/validation.cpp index 214112e2bd..b5d6a66088 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -19,7 +19,6 @@ #include <deploymentstatus.h> #include <flatfile.h> #include <hash.h> -#include <index/blockfilterindex.h> #include <logging.h> #include <logging/timer.h> #include <node/blockstorage.h> @@ -56,30 +55,29 @@ #include <warnings.h> #include <algorithm> +#include <deque> #include <numeric> #include <optional> #include <string> -#include <boost/algorithm/string/replace.hpp> - using node::BLOCKFILE_CHUNK_SIZE; using node::BlockManager; using node::BlockMap; +using node::CBlockIndexHeightOnlyComparator; using node::CBlockIndexWorkComparator; using node::CCoinsStats; using node::CoinStatsHashType; +using node::fImporting; +using node::fPruneMode; +using node::fReindex; using node::GetUTXOStats; +using node::nPruneTarget; using node::OpenBlockFile; using node::ReadBlockFromDisk; using node::SnapshotMetadata; using node::UNDOFILE_CHUNK_SIZE; using node::UndoReadFromDisk; using node::UnlinkPrunedFiles; -using node::fHavePruned; -using node::fImporting; -using node::fPruneMode; -using node::fReindex; -using node::nPruneTarget; #define MICRO 0.000001 #define MILLI 0.001 @@ -106,24 +104,12 @@ const std::vector<std::string> CHECKLEVEL_DOC { "level 4 tries to reconnect the blocks", "each level includes the checks of the previous levels", }; - -bool CBlockIndexWorkComparator::operator()(const CBlockIndex *pa, const CBlockIndex *pb) const { - // First sort by most total work, ... - if (pa->nChainWork > pb->nChainWork) return false; - if (pa->nChainWork < pb->nChainWork) return true; - - // ... then by earliest time received, ... - if (pa->nSequenceId < pb->nSequenceId) return false; - if (pa->nSequenceId > pb->nSequenceId) return true; - - // Use pointer address as tie breaker (should only happen with blocks - // loaded from disk, as those all have id 0). - if (pa < pb) return false; - if (pa > pb) return true; - - // Identical blocks. - return false; -} +/** The number of blocks to keep below the deepest prune lock. + * There is nothing special about this number. It is higher than what we + * expect to see in regular mainnet reorgs, but not so high that it would + * noticeably interfere with the pruning mechanism. + * */ +static constexpr int PRUNE_LOCK_BUFFER{10}; /** * Mutex to guard access to validation specific variables, such as reading @@ -137,7 +123,6 @@ bool CBlockIndexWorkComparator::operator()(const CBlockIndex *pa, const CBlockIn */ RecursiveMutex cs_main; -CBlockIndex *pindexBestHeader = nullptr; Mutex g_best_block_mutex; std::condition_variable g_best_block_cv; uint256 g_best_block; @@ -152,14 +137,14 @@ arith_uint256 nMinimumChainWork; CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); -CBlockIndex* CChainState::FindForkInGlobalIndex(const CBlockLocator& locator) const +const CBlockIndex* CChainState::FindForkInGlobalIndex(const CBlockLocator& locator) const { AssertLockHeld(cs_main); // Find the latest block common to locator and chain - we expect that // locator.vHave is sorted descending by height. for (const uint256& hash : locator.vHave) { - CBlockIndex* pindex{m_blockman.LookupBlockIndex(hash)}; + const CBlockIndex* pindex{m_blockman.LookupBlockIndex(hash)}; if (pindex) { if (m_chain.Contains(pindex)) { return pindex; @@ -178,20 +163,12 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, std::vector<CScriptCheck>* pvChecks = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_main); -bool CheckFinalTx(const CBlockIndex* active_chain_tip, const CTransaction &tx, int flags) +bool CheckFinalTxAtTip(const CBlockIndex* active_chain_tip, const CTransaction& tx) { AssertLockHeld(cs_main); assert(active_chain_tip); // TODO: Make active_chain_tip a reference - // By convention a negative value for flags indicates that the - // current network-enforced consensus rules should be used. In - // a future soft-fork scenario that would mean checking which - // rules would be enforced for the next block and setting the - // appropriate flags. At the present time no soft-forks are - // scheduled, so no flags are set. - flags = std::max(flags, 0); - - // CheckFinalTx() uses active_chain_tip.Height()+1 to evaluate + // CheckFinalTxAtTip() uses active_chain_tip.Height()+1 to evaluate // nLockTime because when IsFinalTx() is called within // AcceptBlock(), the height of the block *being* // evaluated is what is used. Thus if we want to know if a @@ -203,18 +180,15 @@ bool CheckFinalTx(const CBlockIndex* active_chain_tip, const CTransaction &tx, i // less than the median time of the previous block they're contained in. // When the next block is created its previous block will be the current // chain tip, so we use that to calculate the median time passed to - // IsFinalTx() if LOCKTIME_MEDIAN_TIME_PAST is set. - const int64_t nBlockTime = (flags & LOCKTIME_MEDIAN_TIME_PAST) - ? active_chain_tip->GetMedianTimePast() - : GetAdjustedTime(); + // IsFinalTx(). + const int64_t nBlockTime{active_chain_tip->GetMedianTimePast()}; return IsFinalTx(tx, nBlockHeight, nBlockTime); } -bool CheckSequenceLocks(CBlockIndex* tip, +bool CheckSequenceLocksAtTip(CBlockIndex* tip, const CCoinsView& coins_view, const CTransaction& tx, - int flags, LockPoints* lp, bool useExistingLockPoints) { @@ -222,7 +196,7 @@ bool CheckSequenceLocks(CBlockIndex* tip, CBlockIndex index; index.pprev = tip; - // CheckSequenceLocks() uses active_chainstate.m_chain.Height()+1 to evaluate + // CheckSequenceLocksAtTip() uses active_chainstate.m_chain.Height()+1 to evaluate // height based locks because when SequenceLocks() is called within // ConnectBlock(), the height of the block *being* // evaluated is what is used. @@ -252,7 +226,7 @@ bool CheckSequenceLocks(CBlockIndex* tip, prevheights[txinIndex] = coin.nHeight; } } - lockPair = CalculateSequenceLocks(tx, flags, prevheights, index); + lockPair = CalculateSequenceLocks(tx, STANDARD_LOCKTIME_VERIFY_FLAGS, prevheights, index); if (lp) { lp->height = lockPair.first; lp->time = lockPair.second; @@ -268,7 +242,7 @@ bool CheckSequenceLocks(CBlockIndex* tip, // lockPair from CalculateSequenceLocks against tip+1. We know // EvaluateSequenceLocks will fail if there was a non-zero sequence // lock on a mempool input, so we can use the return value of - // CheckSequenceLocks to indicate the LockPoints validity + // CheckSequenceLocksAtTip to indicate the LockPoints validity int maxInputHeight = 0; for (const int height : prevheights) { // Can ignore mempool inputs since we'll fail if they had non-zero locks @@ -276,14 +250,19 @@ bool CheckSequenceLocks(CBlockIndex* tip, maxInputHeight = std::max(maxInputHeight, height); } } - lp->maxInputBlock = tip->GetAncestor(maxInputHeight); + // tip->GetAncestor(maxInputHeight) should never return a nullptr + // because maxInputHeight is always less than the tip height. + // It would, however, be a bad bug to continue execution, since a + // LockPoints object with the maxInputBlock member set to nullptr + // signifies no relative lock time. + lp->maxInputBlock = Assert(tip->GetAncestor(maxInputHeight)); } } return EvaluateSequenceLocks(index, lockPair); } // Returns the script flags which should be checked for a given block -static unsigned int GetBlockScriptFlags(const CBlockIndex* pindex, const Consensus::Params& chainparams); +static unsigned int GetBlockScriptFlags(const CBlockIndex& block_index, const Consensus::Params& chainparams); static void LimitMempoolSize(CTxMemPool& pool, CCoinsViewCache& coins_cache, size_t limit, std::chrono::seconds age) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, pool.cs) @@ -308,8 +287,9 @@ static bool IsCurrentForFeeEstimation(CChainState& active_chainstate) EXCLUSIVE_ return false; if (active_chainstate.m_chain.Tip()->GetBlockTime() < count_seconds(GetTime<std::chrono::seconds>() - MAX_FEE_ESTIMATION_TIP_AGE)) return false; - if (active_chainstate.m_chain.Height() < pindexBestHeader->nHeight - 1) + if (active_chainstate.m_chain.Height() < active_chainstate.m_chainman.m_best_header->nHeight - 1) { return false; + } return true; } @@ -358,26 +338,26 @@ void CChainState::MaybeUpdateMempoolForReorg( // Also updates valid entries' cached LockPoints if needed. // If false, the tx is still valid and its lockpoints are updated. // If true, the tx would be invalid in the next block; remove this entry and all of its descendants. - const auto filter_final_and_mature = [this, flags=STANDARD_LOCKTIME_VERIFY_FLAGS](CTxMemPool::txiter it) + const auto filter_final_and_mature = [this](CTxMemPool::txiter it) EXCLUSIVE_LOCKS_REQUIRED(m_mempool->cs, ::cs_main) { AssertLockHeld(m_mempool->cs); AssertLockHeld(::cs_main); const CTransaction& tx = it->GetTx(); // The transaction must be final. - if (!CheckFinalTx(m_chain.Tip(), tx, flags)) return true; + if (!CheckFinalTxAtTip(m_chain.Tip(), tx)) return true; LockPoints lp = it->GetLockPoints(); const bool validLP{TestLockPointValidity(m_chain, lp)}; CCoinsViewMemPool view_mempool(&CoinsTip(), *m_mempool); - // CheckSequenceLocks checks if the transaction will be final in the next block to be + // CheckSequenceLocksAtTip checks if the transaction will be final in the next block to be // created on top of the new chain. We use useExistingLockPoints=false so that, instead of // using the information in lp (which might now refer to a block that no longer exists in // the chain), it will update lp to contain LockPoints relevant to the new chain. - if (!CheckSequenceLocks(m_chain.Tip(), view_mempool, tx, flags, &lp, validLP)) { - // If CheckSequenceLocks fails, remove the tx and don't depend on the LockPoints. + if (!CheckSequenceLocksAtTip(m_chain.Tip(), view_mempool, tx, &lp, validLP)) { + // If CheckSequenceLocksAtTip fails, remove the tx and don't depend on the LockPoints. return true; } else if (!validLP) { - // If CheckSequenceLocks succeeded, it also updated the LockPoints. + // If CheckSequenceLocksAtTip succeeded, it also updated the LockPoints. // Now update the mempool entry lockpoints as well. m_mempool->mapTx.modify(it, [&lp](CTxMemPoolEntry& e) { e.UpdateLockPoints(lp); }); } @@ -488,6 +468,10 @@ public: * partially submitted. */ const bool m_package_submission; + /** When true, use package feerates instead of individual transaction feerates for fee-based + * policies such as mempool min fee and min relay fee. + */ + const bool m_package_feerates; /** Parameters for single transaction mempool validation. */ static ATMPArgs SingleAccept(const CChainParams& chainparams, int64_t accept_time, @@ -500,6 +484,7 @@ public: /* m_test_accept */ test_accept, /* m_allow_bip125_replacement */ true, /* m_package_submission */ false, + /* m_package_feerates */ false, }; } @@ -513,6 +498,7 @@ public: /* m_test_accept */ true, /* m_allow_bip125_replacement */ false, /* m_package_submission */ false, // not submitting to mempool + /* m_package_feerates */ false, }; } @@ -526,11 +512,44 @@ public: /* m_test_accept */ false, /* m_allow_bip125_replacement */ false, /* m_package_submission */ true, + /* m_package_feerates */ true, + }; + } + + /** Parameters for a single transaction within a package. */ + static ATMPArgs SingleInPackageAccept(const ATMPArgs& package_args) { + return ATMPArgs{/* m_chainparams */ package_args.m_chainparams, + /* m_accept_time */ package_args.m_accept_time, + /* m_bypass_limits */ false, + /* m_coins_to_uncache */ package_args.m_coins_to_uncache, + /* m_test_accept */ package_args.m_test_accept, + /* m_allow_bip125_replacement */ true, + /* m_package_submission */ false, + /* m_package_feerates */ false, // only 1 transaction }; } - // No default ctor to avoid exposing details to clients and allowing the possibility of + + private: + // Private ctor to avoid exposing details to clients and allowing the possibility of // mixing up the order of the arguments. Use static functions above instead. - ATMPArgs() = delete; + ATMPArgs(const CChainParams& chainparams, + int64_t accept_time, + bool bypass_limits, + std::vector<COutPoint>& coins_to_uncache, + bool test_accept, + bool allow_bip125_replacement, + bool package_submission, + bool package_feerates) + : m_chainparams{chainparams}, + m_accept_time{accept_time}, + m_bypass_limits{bypass_limits}, + m_coins_to_uncache{coins_to_uncache}, + m_test_accept{test_accept}, + m_allow_bip125_replacement{allow_bip125_replacement}, + m_package_submission{package_submission}, + m_package_feerates{package_feerates} + { + } }; // Single transaction acceptance @@ -705,8 +724,9 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // Only accept nLockTime-using transactions that can be mined in the next // block; we don't want our mempool filled up with transactions that can't // be mined yet. - if (!CheckFinalTx(m_active_chainstate.m_chain.Tip(), tx, STANDARD_LOCKTIME_VERIFY_FLAGS)) + if (!CheckFinalTxAtTip(m_active_chainstate.m_chain.Tip(), tx)) { return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "non-final"); + } if (m_pool.exists(GenTxid::Wtxid(tx.GetWitnessHash()))) { // Exact transaction already exists in the mempool. @@ -786,8 +806,9 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // be mined yet. // Pass in m_view which has all of the relevant inputs cached. Note that, since m_view's // backend was removed, it no longer pulls coins from the mempool. - if (!CheckSequenceLocks(m_active_chainstate.m_chain.Tip(), m_view, tx, STANDARD_LOCKTIME_VERIFY_FLAGS, &lp)) + if (!CheckSequenceLocksAtTip(m_active_chainstate.m_chain.Tip(), m_view, tx, &lp)) { return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "non-BIP68-final"); + } // The mempool holds txs for the next block, so pass height+1 to CheckTxInputs if (!Consensus::CheckTxInputs(tx, state, m_view, m_active_chainstate.m_chain.Height() + 1, ws.m_base_fees)) { @@ -828,9 +849,10 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) return state.Invalid(TxValidationResult::TX_NOT_STANDARD, "bad-txns-too-many-sigops", strprintf("%d", nSigOpsCost)); - // No transactions are allowed below minRelayTxFee except from disconnected - // blocks - if (!bypass_limits && !CheckFeeRate(ws.m_vsize, ws.m_modified_fees, state)) return false; + // No individual transactions are allowed below minRelayTxFee and mempool min fee except from + // disconnected blocks and transactions in a package. Package transactions will be checked using + // package feerate later. + if (!bypass_limits && !args.m_package_feerates && !CheckFeeRate(ws.m_vsize, ws.m_modified_fees, state)) return false; ws.m_iters_conflicting = m_pool.GetIterSet(ws.m_conflicts); // Calculate in-mempool ancestors, up to a limit. @@ -1022,7 +1044,7 @@ bool MemPoolAccept::ConsensusScriptChecks(const ATMPArgs& args, Workspace& ws) // There is a similar check in CreateNewBlock() to prevent creating // invalid blocks (using TestBlockValidity), however allowing such // transactions into the mempool can be exploited as a DoS attack. - unsigned int currentBlockScriptVerifyFlags = GetBlockScriptFlags(m_active_chainstate.m_chain.Tip(), chainparams.GetConsensus()); + unsigned int currentBlockScriptVerifyFlags{GetBlockScriptFlags(*m_active_chainstate.m_chain.Tip(), chainparams.GetConsensus())}; if (!CheckInputsFromMempoolAndCache(tx, state, m_view, m_pool, currentBlockScriptVerifyFlags, ws.m_precomputed_txdata, m_active_chainstate.CoinsTip())) { LogPrintf("BUG! PLEASE REPORT THIS! CheckInputScripts failed against latest-block but not STANDARD flags %s, %s\n", hash.ToString(), state.ToString()); @@ -1098,6 +1120,7 @@ bool MemPoolAccept::SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& if (!ConsensusScriptChecks(args, ws)) { results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Failure(ws.m_state)); // Since PolicyScriptChecks() passed, this should never fail. + Assume(false); all_submitted = false; package_state.Invalid(PackageValidationResult::PCKG_MEMPOOL_ERROR, strprintf("BUG! PolicyScriptChecks succeeded but ConsensusScriptChecks failed: %s", @@ -1112,6 +1135,7 @@ bool MemPoolAccept::SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& m_limit_descendant_size, unused_err_string)) { results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Failure(ws.m_state)); // Since PreChecks() and PackageMempoolChecks() both enforce limits, this should never fail. + Assume(false); all_submitted = false; package_state.Invalid(PackageValidationResult::PCKG_MEMPOOL_ERROR, strprintf("BUG! Mempool ancestors or descendants were underestimated: %s", @@ -1125,6 +1149,7 @@ bool MemPoolAccept::SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& if (!Finalize(args, ws)) { results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Failure(ws.m_state)); // Since LimitMempoolSize() won't be called, this should never fail. + Assume(false); all_submitted = false; package_state.Invalid(PackageValidationResult::PCKG_MEMPOOL_ERROR, strprintf("BUG! Adding to mempool failed: %s", ws.m_ptx->GetHash().ToString())); @@ -1214,12 +1239,27 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: m_viewmempool.PackageAddTransaction(ws.m_ptx); } + // Transactions must meet two minimum feerates: the mempool minimum fee and min relay fee. + // For transactions consisting of exactly one child and its parents, it suffices to use the + // package feerate (total modified fees / total virtual size) to check this requirement. + const auto m_total_vsize = std::accumulate(workspaces.cbegin(), workspaces.cend(), int64_t{0}, + [](int64_t sum, auto& ws) { return sum + ws.m_vsize; }); + const auto m_total_modified_fees = std::accumulate(workspaces.cbegin(), workspaces.cend(), CAmount{0}, + [](CAmount sum, auto& ws) { return sum + ws.m_modified_fees; }); + const CFeeRate package_feerate(m_total_modified_fees, m_total_vsize); + TxValidationState placeholder_state; + if (args.m_package_feerates && + !CheckFeeRate(m_total_vsize, m_total_modified_fees, placeholder_state)) { + package_state.Invalid(PackageValidationResult::PCKG_POLICY, "package-fee-too-low"); + return PackageMempoolAcceptResult(package_state, package_feerate, {}); + } + // Apply package mempool ancestor/descendant limits. Skip if there is only one transaction, // because it's unnecessary. Also, CPFP carve out can increase the limit for individual // transactions, but this exemption is not extended to packages in CheckPackageLimits(). std::string err_string; if (txns.size() > 1 && !PackageMempoolChecks(txns, package_state)) { - return PackageMempoolAcceptResult(package_state, std::move(results)); + return PackageMempoolAcceptResult(package_state, package_feerate, std::move(results)); } for (Workspace& ws : workspaces) { @@ -1227,7 +1267,7 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: // Exit early to avoid doing pointless work. Update the failed tx result; the rest are unfinished. package_state.Invalid(PackageValidationResult::PCKG_TX, "transaction failed"); results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Failure(ws.m_state)); - return PackageMempoolAcceptResult(package_state, std::move(results)); + return PackageMempoolAcceptResult(package_state, package_feerate, std::move(results)); } if (args.m_test_accept) { // When test_accept=true, transactions that pass PolicyScriptChecks are valid because there are @@ -1238,14 +1278,14 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: } } - if (args.m_test_accept) return PackageMempoolAcceptResult(package_state, std::move(results)); + if (args.m_test_accept) return PackageMempoolAcceptResult(package_state, package_feerate, std::move(results)); if (!SubmitPackage(args, workspaces, package_state, results)) { // PackageValidationState filled in by SubmitPackage(). - return PackageMempoolAcceptResult(package_state, std::move(results)); + return PackageMempoolAcceptResult(package_state, package_feerate, std::move(results)); } - return PackageMempoolAcceptResult(package_state, std::move(results)); + return PackageMempoolAcceptResult(package_state, package_feerate, std::move(results)); } PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, ATMPArgs& args) @@ -1312,6 +1352,8 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, // transactions that are already in the mempool, and only call AcceptMultipleTransactions() with // the new transactions. This ensures we don't double-count transaction counts and sizes when // checking ancestor/descendant limits, or double-count transaction fees for fee-related policy. + ATMPArgs single_args = ATMPArgs::SingleInPackageAccept(args); + bool quit_early{false}; std::vector<CTransactionRef> txns_new; for (const auto& tx : package) { const auto& wtxid = tx->GetWitnessHash(); @@ -1338,18 +1380,43 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, results.emplace(wtxid, MempoolAcceptResult::MempoolTxDifferentWitness(iter.value()->GetTx().GetWitnessHash())); } else { // Transaction does not already exist in the mempool. - txns_new.push_back(tx); + // Try submitting the transaction on its own. + const auto single_res = AcceptSingleTransaction(tx, single_args); + if (single_res.m_result_type == MempoolAcceptResult::ResultType::VALID) { + // The transaction succeeded on its own and is now in the mempool. Don't include it + // in package validation, because its fees should only be "used" once. + assert(m_pool.exists(GenTxid::Wtxid(wtxid))); + results.emplace(wtxid, single_res); + } else if (single_res.m_state.GetResult() != TxValidationResult::TX_MEMPOOL_POLICY && + single_res.m_state.GetResult() != TxValidationResult::TX_MISSING_INPUTS) { + // Package validation policy only differs from individual policy in its evaluation + // of feerate. For example, if a transaction fails here due to violation of a + // consensus rule, the result will not change when it is submitted as part of a + // package. To minimize the amount of repeated work, unless the transaction fails + // due to feerate or missing inputs (its parent is a previous transaction in the + // package that failed due to feerate), don't run package validation. Note that this + // decision might not make sense if different types of packages are allowed in the + // future. Continue individually validating the rest of the transactions, because + // some of them may still be valid. + quit_early = true; + } else { + txns_new.push_back(tx); + } } } // Nothing to do if the entire package has already been submitted. - if (txns_new.empty()) return PackageMempoolAcceptResult(package_state, std::move(results)); + if (quit_early || txns_new.empty()) { + // No package feerate when no package validation was done. + return PackageMempoolAcceptResult(package_state, std::move(results)); + } // Validate the (deduplicated) transactions as a package. auto submission_result = AcceptMultipleTransactions(txns_new, args); // Include already-in-mempool transaction results in the final result. for (const auto& [wtxid, mempoolaccept_res] : results) { submission_result.m_tx_results.emplace(wtxid, mempoolaccept_res); } + if (submission_result.m_state.IsValid()) assert(submission_result.m_package_feerate.has_value()); return submission_result; } @@ -1428,7 +1495,7 @@ CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams) } CoinsViews::CoinsViews( - std::string ldb_name, + fs::path ldb_name, size_t cache_size_bytes, bool in_memory, bool should_wipe) : m_dbview( @@ -1456,7 +1523,7 @@ void CChainState::InitCoinsDB( size_t cache_size_bytes, bool in_memory, bool should_wipe, - std::string leveldb_name) + fs::path leveldb_name) { if (m_from_snapshot_blockhash) { leveldb_name += "_" + m_from_snapshot_blockhash->ToString(); @@ -1514,7 +1581,7 @@ static void AlertNotify(const std::string& strMessage) std::string singleQuote("'"); std::string safeStatus = SanitizeString(strMessage); safeStatus = singleQuote+safeStatus+singleQuote; - boost::replace_all(strCmd, "%s", safeStatus); + ReplaceAll(strCmd, "%s", safeStatus); std::thread t(runCommand, strCmd); t.detach(); // thread runs free @@ -1546,8 +1613,8 @@ void CChainState::InvalidChainFound(CBlockIndex* pindexNew) if (!m_chainman.m_best_invalid || pindexNew->nChainWork > m_chainman.m_best_invalid->nChainWork) { m_chainman.m_best_invalid = pindexNew; } - if (pindexBestHeader != nullptr && pindexBestHeader->GetAncestor(pindexNew->nHeight) == pindexNew) { - pindexBestHeader = m_chain.Tip(); + if (m_chainman.m_best_header != nullptr && m_chainman.m_best_header->GetAncestor(pindexNew->nHeight) == pindexNew) { + m_chainman.m_best_header = m_chain.Tip(); } LogPrintf("%s: invalid block=%s height=%d log2_work=%f date=%s\n", __func__, @@ -1861,47 +1928,41 @@ public: } }; -static ThresholdConditionCache warningcache[VERSIONBITS_NUM_BITS] GUARDED_BY(cs_main); +static std::array<ThresholdConditionCache, VERSIONBITS_NUM_BITS> warningcache GUARDED_BY(cs_main); -static unsigned int GetBlockScriptFlags(const CBlockIndex* pindex, const Consensus::Params& consensusparams) +static unsigned int GetBlockScriptFlags(const CBlockIndex& block_index, const Consensus::Params& consensusparams) { - unsigned int flags = SCRIPT_VERIFY_NONE; - // BIP16 didn't become active until Apr 1 2012 (on mainnet, and // retroactively applied to testnet) // However, only one historical block violated the P2SH rules (on both - // mainnet and testnet), so for simplicity, always leave P2SH - // on except for the one violating block. - if (consensusparams.BIP16Exception.IsNull() || // no bip16 exception on this chain - pindex->phashBlock == nullptr || // this is a new candidate block, eg from TestBlockValidity() - *pindex->phashBlock != consensusparams.BIP16Exception) // this block isn't the historical exception - { - // Enforce WITNESS rules whenever P2SH is in effect - flags |= SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS; + // mainnet and testnet). + // Similarly, only one historical block violated the TAPROOT rules on + // mainnet. + // For simplicity, always leave P2SH+WITNESS+TAPROOT on except for the two + // violating blocks. + uint32_t flags{SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_TAPROOT}; + const auto it{consensusparams.script_flag_exceptions.find(*Assert(block_index.phashBlock))}; + if (it != consensusparams.script_flag_exceptions.end()) { + flags = it->second; } // Enforce the DERSIG (BIP66) rule - if (DeploymentActiveAt(*pindex, consensusparams, Consensus::DEPLOYMENT_DERSIG)) { + if (DeploymentActiveAt(block_index, consensusparams, Consensus::DEPLOYMENT_DERSIG)) { flags |= SCRIPT_VERIFY_DERSIG; } // Enforce CHECKLOCKTIMEVERIFY (BIP65) - if (DeploymentActiveAt(*pindex, consensusparams, Consensus::DEPLOYMENT_CLTV)) { + if (DeploymentActiveAt(block_index, consensusparams, Consensus::DEPLOYMENT_CLTV)) { flags |= SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY; } // Enforce CHECKSEQUENCEVERIFY (BIP112) - if (DeploymentActiveAt(*pindex, consensusparams, Consensus::DEPLOYMENT_CSV)) { + if (DeploymentActiveAt(block_index, consensusparams, Consensus::DEPLOYMENT_CSV)) { flags |= SCRIPT_VERIFY_CHECKSEQUENCEVERIFY; } - // Enforce Taproot (BIP340-BIP342) - if (DeploymentActiveAt(*pindex, consensusparams, Consensus::DEPLOYMENT_TAPROOT)) { - flags |= SCRIPT_VERIFY_TAPROOT; - } - // Enforce BIP147 NULLDUMMY (activated simultaneously with segwit) - if (DeploymentActiveAt(*pindex, consensusparams, Consensus::DEPLOYMENT_SEGWIT)) { + if (DeploymentActiveAt(block_index, consensusparams, Consensus::DEPLOYMENT_SEGWIT)) { flags |= SCRIPT_VERIFY_NULLDUMMY; } @@ -1909,11 +1970,11 @@ static unsigned int GetBlockScriptFlags(const CBlockIndex* pindex, const Consens } - static int64_t nTimeCheck = 0; static int64_t nTimeForks = 0; -static int64_t nTimeVerify = 0; static int64_t nTimeConnect = 0; +static int64_t nTimeVerify = 0; +static int64_t nTimeUndo = 0; static int64_t nTimeIndex = 0; static int64_t nTimeTotal = 0; static int64_t nBlocksTotal = 0; @@ -1978,9 +2039,9 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, // effectively caching the result of part of the verification. BlockMap::const_iterator it = m_blockman.m_block_index.find(hashAssumeValid); if (it != m_blockman.m_block_index.end()) { - if (it->second->GetAncestor(pindex->nHeight) == pindex && - pindexBestHeader->GetAncestor(pindex->nHeight) == pindex && - pindexBestHeader->nChainWork >= nMinimumChainWork) { + if (it->second.GetAncestor(pindex->nHeight) == pindex && + m_chainman.m_best_header->GetAncestor(pindex->nHeight) == pindex && + m_chainman.m_best_header->nChainWork >= nMinimumChainWork) { // This block is a member of the assumed verified chain and an ancestor of the best header. // Script verification is skipped when connecting blocks under the // assumevalid block. Assuming the assumevalid block is valid this @@ -1995,7 +2056,7 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, // artificially set the default assumed verified block further back. // The test against nMinimumChainWork prevents the skipping when denied access to any chain at // least as good as the expected chain. - fScriptChecks = (GetBlockProofEquivalentTime(*pindexBestHeader, *pindex, *pindexBestHeader, m_params.GetConsensus()) <= 60 * 60 * 24 * 7 * 2); + fScriptChecks = (GetBlockProofEquivalentTime(*m_chainman.m_best_header, *pindex, *m_chainman.m_best_header, m_params.GetConsensus()) <= 60 * 60 * 24 * 7 * 2); } } } @@ -2097,7 +2158,7 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, } // Get the script flags for this block - unsigned int flags = GetBlockScriptFlags(pindex, m_params.GetConsensus()); + unsigned int flags{GetBlockScriptFlags(*pindex, m_params.GetConsensus())}; int64_t nTime2 = GetTimeMicros(); nTimeForks += nTime2 - nTime1; LogPrint(BCLog::BENCH, " - Fork checks: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime2 - nTime1), nTimeForks * MICRO, nTimeForks * MILLI / nBlocksTotal); @@ -2207,6 +2268,9 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, return false; } + int64_t nTime5 = GetTimeMicros(); nTimeUndo += nTime5 - nTime4; + LogPrint(BCLog::BENCH, " - Write undo data: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime5 - nTime4), nTimeUndo * MICRO, nTimeUndo * MILLI / nBlocksTotal); + if (!pindex->IsValid(BLOCK_VALID_SCRIPTS)) { pindex->RaiseValidity(BLOCK_VALID_SCRIPTS); m_blockman.m_dirty_blockindex.insert(pindex); @@ -2216,8 +2280,8 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, // add this block to the view's block chain view.SetBestBlock(pindex->GetBlockHash()); - int64_t nTime5 = GetTimeMicros(); nTimeIndex += nTime5 - nTime4; - LogPrint(BCLog::BENCH, " - Index writing: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime5 - nTime4), nTimeIndex * MICRO, nTimeIndex * MILLI / nBlocksTotal); + int64_t nTime6 = GetTimeMicros(); nTimeIndex += nTime6 - nTime5; + LogPrint(BCLog::BENCH, " - Index writing: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime6 - nTime5), nTimeIndex * MICRO, nTimeIndex * MILLI / nBlocksTotal); TRACE6(validation, block_connected, block_hash.data(), @@ -2286,12 +2350,24 @@ bool CChainState::FlushStateToDisk( CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(); LOCK(m_blockman.cs_LastBlockFile); if (fPruneMode && (m_blockman.m_check_for_pruning || nManualPruneHeight > 0) && !fReindex) { - // make sure we don't prune above the blockfilterindexes bestblocks + // make sure we don't prune above any of the prune locks bestblocks // pruning is height-based - int last_prune = m_chain.Height(); // last height we can prune - ForEachBlockFilterIndex([&](BlockFilterIndex& index) { - last_prune = std::max(1, std::min(last_prune, index.GetSummary().best_block_height)); - }); + int last_prune{m_chain.Height()}; // last height we can prune + std::optional<std::string> limiting_lock; // prune lock that actually was the limiting factor, only used for logging + + for (const auto& prune_lock : m_blockman.m_prune_locks) { + if (prune_lock.second.height_first == std::numeric_limits<int>::max()) continue; + // Remove the buffer and one additional block here to get actual height that is outside of the buffer + const int lock_height{prune_lock.second.height_first - PRUNE_LOCK_BUFFER - 1}; + last_prune = std::max(1, std::min(last_prune, lock_height)); + if (last_prune == lock_height) { + limiting_lock = prune_lock.first; + } + } + + if (limiting_lock) { + LogPrint(BCLog::PRUNE, "%s limited pruning to height %d\n", limiting_lock.value(), last_prune); + } if (nManualPruneHeight > 0) { LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune (manual)", BCLog::BENCH); @@ -2305,9 +2381,9 @@ bool CChainState::FlushStateToDisk( } if (!setFilesToPrune.empty()) { fFlushForPrune = true; - if (!fHavePruned) { + if (!m_blockman.m_have_pruned) { m_blockman.m_block_tree_db->WriteFlag("prunedblockfiles", true); - fHavePruned = true; + m_blockman.m_have_pruned = true; } } } @@ -2481,7 +2557,7 @@ void CChainState::UpdateTip(const CBlockIndex* pindexNew) const CBlockIndex* pindex = pindexNew; for (int bit = 0; bit < VERSIONBITS_NUM_BITS; bit++) { WarningBitsConditionChecker checker(bit); - ThresholdState state = checker.GetStateFor(pindex, m_params.GetConsensus(), warningcache[bit]); + ThresholdState state = checker.GetStateFor(pindex, m_params.GetConsensus(), warningcache.at(bit)); if (state == ThresholdState::ACTIVE || state == ThresholdState::LOCKED_IN) { const bilingual_str warning = strprintf(_("Unknown new rules activated (versionbit %i)"), bit); if (state == ThresholdState::ACTIVE) { @@ -2529,6 +2605,18 @@ bool CChainState::DisconnectTip(BlockValidationState& state, DisconnectedBlockTr assert(flushed); } LogPrint(BCLog::BENCH, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * MILLI); + + { + // Prune locks that began at or after the tip should be moved backward so they get a chance to reorg + const int max_height_first{pindexDelete->nHeight - 1}; + for (auto& prune_lock : m_blockman.m_prune_locks) { + if (prune_lock.second.height_first <= max_height_first) continue; + + prune_lock.second.height_first = max_height_first; + LogPrint(BCLog::PRUNE, "%s prune lock moved back to %d\n", prune_lock.first, max_height_first); + } + } + // Write the chain state to disk, if necessary. if (!FlushStateToDisk(state, FlushStateMode::IF_NEEDED)) { return false; @@ -2556,7 +2644,7 @@ bool CChainState::DisconnectTip(BlockValidationState& state, DisconnectedBlockTr return true; } -static int64_t nTimeReadFromDisk = 0; +static int64_t nTimeReadFromDiskTotal = 0; static int64_t nTimeConnectTotal = 0; static int64_t nTimeFlush = 0; static int64_t nTimeChainState = 0; @@ -2624,13 +2712,14 @@ bool CChainState::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew } pthisBlock = pblockNew; } else { + LogPrint(BCLog::BENCH, " - Using cached block\n"); pthisBlock = pblock; } const CBlock& blockConnecting = *pthisBlock; // Apply the block atomically to the chain state. - int64_t nTime2 = GetTimeMicros(); nTimeReadFromDisk += nTime2 - nTime1; + int64_t nTime2 = GetTimeMicros(); nTimeReadFromDiskTotal += nTime2 - nTime1; int64_t nTime3; - LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * MILLI, nTimeReadFromDisk * MICRO); + LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime2 - nTime1) * MILLI, nTimeReadFromDiskTotal * MICRO, nTimeReadFromDiskTotal * MILLI / nBlocksTotal); { CCoinsViewCache view(&CoinsTip()); bool rv = ConnectBlock(blockConnecting, state, pindexNew, view); @@ -2850,7 +2939,7 @@ static bool NotifyHeaderTip(CChainState& chainstate) LOCKS_EXCLUDED(cs_main) { CBlockIndex* pindexHeader = nullptr; { LOCK(cs_main); - pindexHeader = pindexBestHeader; + pindexHeader = chainstate.m_chainman.m_best_header; if (pindexHeader != pindexHeaderOld) { fNotify = true; @@ -3035,8 +3124,8 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, CBlockIndex* pind { LOCK(cs_main); - for (const auto& entry : m_blockman.m_block_index) { - CBlockIndex *candidate = entry.second; + for (auto& entry : m_blockman.m_block_index) { + CBlockIndex* candidate = &entry.second; // We don't need to put anything in our active chain into the // multimap, because those candidates will be found and considered // as we disconnect. @@ -3133,12 +3222,10 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, CBlockIndex* pind // it up here, this should be an essentially unobservable error. // Loop back over all block index entries and add any missing entries // to setBlockIndexCandidates. - BlockMap::iterator it = m_blockman.m_block_index.begin(); - while (it != m_blockman.m_block_index.end()) { - if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->HaveTxsDownloaded() && !setBlockIndexCandidates.value_comp()(it->second, m_chain.Tip())) { - setBlockIndexCandidates.insert(it->second); + for (auto& [_, block_index] : m_blockman.m_block_index) { + if (block_index.IsValid(BLOCK_VALID_TRANSACTIONS) && block_index.HaveTxsDownloaded() && !setBlockIndexCandidates.value_comp()(&block_index, m_chain.Tip())) { + setBlockIndexCandidates.insert(&block_index); } - it++; } InvalidChainFound(to_mark_failed); @@ -3157,21 +3244,19 @@ void CChainState::ResetBlockFailureFlags(CBlockIndex *pindex) { int nHeight = pindex->nHeight; // Remove the invalidity flag from this block and all its descendants. - BlockMap::iterator it = m_blockman.m_block_index.begin(); - while (it != m_blockman.m_block_index.end()) { - if (!it->second->IsValid() && it->second->GetAncestor(nHeight) == pindex) { - it->second->nStatus &= ~BLOCK_FAILED_MASK; - m_blockman.m_dirty_blockindex.insert(it->second); - if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->HaveTxsDownloaded() && setBlockIndexCandidates.value_comp()(m_chain.Tip(), it->second)) { - setBlockIndexCandidates.insert(it->second); + for (auto& [_, block_index] : m_blockman.m_block_index) { + if (!block_index.IsValid() && block_index.GetAncestor(nHeight) == pindex) { + block_index.nStatus &= ~BLOCK_FAILED_MASK; + m_blockman.m_dirty_blockindex.insert(&block_index); + if (block_index.IsValid(BLOCK_VALID_TRANSACTIONS) && block_index.HaveTxsDownloaded() && setBlockIndexCandidates.value_comp()(m_chain.Tip(), &block_index)) { + setBlockIndexCandidates.insert(&block_index); } - if (it->second == m_chainman.m_best_invalid) { + if (&block_index == m_chainman.m_best_invalid) { // Reset invalid block marker if it was pointing to one of those. m_chainman.m_best_invalid = nullptr; } - m_chainman.m_failed_blocks.erase(it->second); + m_chainman.m_failed_blocks.erase(&block_index); } - it++; } // Remove the invalidity flag from all ancestors too. @@ -3377,7 +3462,7 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidatio // Don't accept any forks from the main chain prior to last checkpoint. // GetLastCheckpoint finds the last checkpoint in MapCheckpoints that's in our // BlockIndex(). - CBlockIndex* pcheckpoint = blockman.GetLastCheckpoint(params.Checkpoints()); + const CBlockIndex* pcheckpoint = blockman.GetLastCheckpoint(params.Checkpoints()); if (pcheckpoint && nHeight < pcheckpoint->nHeight) { LogPrintf("ERROR: %s: forked chain older than last checkpoint (height %d)\n", __func__, nHeight); return state.Invalid(BlockValidationResult::BLOCK_CHECKPOINT, "bad-fork-prior-to-checkpoint"); @@ -3500,7 +3585,7 @@ bool ChainstateManager::AcceptBlockHeader(const CBlockHeader& block, BlockValida if (hash != chainparams.GetConsensus().hashGenesisBlock) { if (miSelf != m_blockman.m_block_index.end()) { // Block header is already known. - CBlockIndex* pindex = miSelf->second; + CBlockIndex* pindex = &(miSelf->second); if (ppindex) *ppindex = pindex; if (pindex->nStatus & BLOCK_FAILED_MASK) { @@ -3522,7 +3607,7 @@ bool ChainstateManager::AcceptBlockHeader(const CBlockHeader& block, BlockValida LogPrint(BCLog::VALIDATION, "%s: %s prev block not found\n", __func__, hash.ToString()); return state.Invalid(BlockValidationResult::BLOCK_MISSING_PREV, "prev-blk-not-found"); } - pindexPrev = (*mi).second; + pindexPrev = &((*mi).second); if (pindexPrev->nStatus & BLOCK_FAILED_MASK) { LogPrint(BCLog::VALIDATION, "%s: %s prev block invalid\n", __func__, hash.ToString()); return state.Invalid(BlockValidationResult::BLOCK_INVALID_PREV, "bad-prevblk"); @@ -3571,7 +3656,7 @@ bool ChainstateManager::AcceptBlockHeader(const CBlockHeader& block, BlockValida } } } - CBlockIndex* pindex{m_blockman.AddToBlockIndex(block)}; + CBlockIndex* pindex{m_blockman.AddToBlockIndex(block, m_best_header)}; if (ppindex) *ppindex = pindex; @@ -3994,13 +4079,13 @@ bool CChainState::ReplayBlocks() if (m_blockman.m_block_index.count(hashHeads[0]) == 0) { return error("ReplayBlocks(): reorganization to unknown block requested"); } - pindexNew = m_blockman.m_block_index[hashHeads[0]]; + pindexNew = &(m_blockman.m_block_index[hashHeads[0]]); if (!hashHeads[1].IsNull()) { // The old tip is allowed to be 0, indicating it's the first flush. if (m_blockman.m_block_index.count(hashHeads[1]) == 0) { return error("ReplayBlocks(): reorganization from unknown block requested"); } - pindexOld = m_blockman.m_block_index[hashHeads[1]]; + pindexOld = &(m_blockman.m_block_index[hashHeads[1]]); pindexFork = LastCommonAncestor(pindexOld, pindexNew); assert(pindexFork != nullptr); } @@ -4028,10 +4113,11 @@ bool CChainState::ReplayBlocks() // Roll forward from the forking point to the new tip. int nForkHeight = pindexFork ? pindexFork->nHeight : 0; for (int nHeight = nForkHeight + 1; nHeight <= pindexNew->nHeight; ++nHeight) { - const CBlockIndex* pindex = pindexNew->GetAncestor(nHeight); - LogPrintf("Rolling forward %s (%i)\n", pindex->GetBlockHash().ToString(), nHeight); + const CBlockIndex& pindex{*Assert(pindexNew->GetAncestor(nHeight))}; + + LogPrintf("Rolling forward %s (%i)\n", pindex.GetBlockHash().ToString(), nHeight); uiInterface.ShowProgress(_("Replaying blocks…").translated, (int) ((nHeight - nForkHeight) * 100.0 / (pindexNew->nHeight - nForkHeight)) , false); - if (!RollforwardBlock(pindex, cache)) return false; + if (!RollforwardBlock(&pindex, cache)) return false; } cache.SetBestBlock(pindexNew->GetBlockHash()); @@ -4065,30 +4151,82 @@ void CChainState::UnloadBlockIndex() setBlockIndexCandidates.clear(); } -// May NOT be used after any connections are up as much -// of the peer-processing logic assumes a consistent -// block index state -void UnloadBlockIndex(CTxMemPool* mempool, ChainstateManager& chainman) -{ - LOCK(cs_main); - chainman.Unload(); - pindexBestHeader = nullptr; - if (mempool) mempool->clear(); - g_versionbitscache.Clear(); - for (int b = 0; b < VERSIONBITS_NUM_BITS; b++) { - warningcache[b].clear(); - } - fHavePruned = false; -} - bool ChainstateManager::LoadBlockIndex() { AssertLockHeld(cs_main); // Load block index from databases bool needs_init = fReindex; if (!fReindex) { - bool ret = m_blockman.LoadBlockIndexDB(*this); + bool ret = m_blockman.LoadBlockIndexDB(); if (!ret) return false; + + std::vector<CBlockIndex*> vSortedByHeight{m_blockman.GetAllBlockIndices()}; + std::sort(vSortedByHeight.begin(), vSortedByHeight.end(), + CBlockIndexHeightOnlyComparator()); + + // Find start of assumed-valid region. + int first_assumed_valid_height = std::numeric_limits<int>::max(); + + for (const CBlockIndex* block : vSortedByHeight) { + if (block->IsAssumedValid()) { + auto chainstates = GetAll(); + + // If we encounter an assumed-valid block index entry, ensure that we have + // one chainstate that tolerates assumed-valid entries and another that does + // not (i.e. the background validation chainstate), since assumed-valid + // entries should always be pending validation by a fully-validated chainstate. + auto any_chain = [&](auto fnc) { return std::any_of(chainstates.cbegin(), chainstates.cend(), fnc); }; + assert(any_chain([](auto chainstate) { return chainstate->reliesOnAssumedValid(); })); + assert(any_chain([](auto chainstate) { return !chainstate->reliesOnAssumedValid(); })); + + first_assumed_valid_height = block->nHeight; + break; + } + } + + for (CBlockIndex* pindex : vSortedByHeight) { + if (ShutdownRequested()) return false; + if (pindex->IsAssumedValid() || + (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && + (pindex->HaveTxsDownloaded() || pindex->pprev == nullptr))) { + + // Fill each chainstate's block candidate set. Only add assumed-valid + // blocks to the tip candidate set if the chainstate is allowed to rely on + // assumed-valid blocks. + // + // If all setBlockIndexCandidates contained the assumed-valid blocks, the + // background chainstate's ActivateBestChain() call would add assumed-valid + // blocks to the chain (based on how FindMostWorkChain() works). Obviously + // we don't want this since the purpose of the background validation chain + // is to validate assued-valid blocks. + // + // Note: This is considering all blocks whose height is greater or equal to + // the first assumed-valid block to be assumed-valid blocks, and excluding + // them from the background chainstate's setBlockIndexCandidates set. This + // does mean that some blocks which are not technically assumed-valid + // (later blocks on a fork beginning before the first assumed-valid block) + // might not get added to the background chainstate, but this is ok, + // because they will still be attached to the active chainstate if they + // actually contain more work. + // + // Instead of this height-based approach, an earlier attempt was made at + // detecting "holistically" whether the block index under consideration + // relied on an assumed-valid ancestor, but this proved to be too slow to + // be practical. + for (CChainState* chainstate : GetAll()) { + if (chainstate->reliesOnAssumedValid() || + pindex->nHeight < first_assumed_valid_height) { + chainstate->setBlockIndexCandidates.insert(pindex); + } + } + } + if (pindex->nStatus & BLOCK_FAILED_MASK && (!m_best_invalid || pindex->nChainWork > m_best_invalid->nChainWork)) { + m_best_invalid = pindex; + } + if (pindex->IsValid(BLOCK_VALID_TREE) && (m_best_header == nullptr || CBlockIndexWorkComparator()(m_best_header, pindex))) + m_best_header = pindex; + } + needs_init = m_blockman.m_block_index.empty(); } @@ -4121,7 +4259,7 @@ bool CChainState::LoadGenesisBlock() if (blockPos.IsNull()) { return error("%s: writing genesis block to disk failed", __func__); } - CBlockIndex *pindex = m_blockman.AddToBlockIndex(block); + CBlockIndex* pindex = m_blockman.AddToBlockIndex(block, m_chainman.m_best_header); ReceivedBlockTransactions(block, pindex, blockPos); } catch (const std::runtime_error& e) { return error("%s: failed to write genesis block: %s", __func__, e.what()); @@ -4190,7 +4328,7 @@ void CChainState::LoadExternalBlockFile(FILE* fileIn, FlatFilePos* dbp) } // process in case the block isn't known yet - CBlockIndex* pindex = m_blockman.LookupBlockIndex(hash); + const CBlockIndex* pindex = m_blockman.LookupBlockIndex(hash); if (!pindex || (pindex->nStatus & BLOCK_HAVE_DATA) == 0) { BlockValidationState state; if (AcceptBlock(pblock, state, nullptr, true, dbp, nullptr)) { @@ -4267,8 +4405,8 @@ void CChainState::CheckBlockIndex() // Build forward-pointing map of the entire block tree. std::multimap<CBlockIndex*,CBlockIndex*> forward; - for (const std::pair<const uint256, CBlockIndex*>& entry : m_blockman.m_block_index) { - forward.insert(std::make_pair(entry.second->pprev, entry.second)); + for (auto& [_, block_index] : m_blockman.m_block_index) { + forward.emplace(block_index.pprev, &block_index); } assert(forward.size() == m_blockman.m_block_index.size()); @@ -4332,7 +4470,7 @@ void CChainState::CheckBlockIndex() // HAVE_DATA is only equivalent to nTx > 0 (or VALID_TRANSACTIONS) if no pruning has occurred. // Unless these indexes are assumed valid and pending block download on a // background chainstate. - if (!fHavePruned && !pindex->IsAssumedValid()) { + if (!m_blockman.m_have_pruned && !pindex->IsAssumedValid()) { // If we've never pruned, then HAVE_DATA should be equivalent to nTx > 0 assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0)); assert(pindexFirstMissing == pindexFirstNeverProcessed); @@ -4406,7 +4544,7 @@ void CChainState::CheckBlockIndex() if (pindexFirstMissing == nullptr) assert(!foundInUnlinked); // We aren't missing data for any parent -- cannot be in m_blocks_unlinked. if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed == nullptr && pindexFirstMissing != nullptr) { // We HAVE_DATA for this block, have received data for all parents at some point, but we're currently missing data for some parent. - assert(fHavePruned); // We must have pruned. + assert(m_blockman.m_have_pruned); // We must have pruned. // This block may have entered m_blocks_unlinked if: // - it has a descendant that at some point had more work than the // tip, and @@ -4786,7 +4924,7 @@ bool ChainstateManager::ActivateSnapshot( auto snapshot_chainstate = WITH_LOCK(::cs_main, return std::make_unique<CChainState>( - /* mempool */ nullptr, m_blockman, *this, base_blockhash)); + /*mempool=*/nullptr, m_blockman, *this, base_blockhash)); { LOCK(::cs_main); @@ -5043,28 +5181,6 @@ bool ChainstateManager::IsSnapshotActive() const return m_snapshot_chainstate && m_active_chainstate == m_snapshot_chainstate.get(); } -void ChainstateManager::Unload() -{ - AssertLockHeld(::cs_main); - for (CChainState* chainstate : this->GetAll()) { - chainstate->m_chain.SetTip(nullptr); - chainstate->UnloadBlockIndex(); - } - - m_failed_blocks.clear(); - m_blockman.Unload(); - m_best_invalid = nullptr; -} - -void ChainstateManager::Reset() -{ - LOCK(::cs_main); - m_ibd_chainstate.reset(); - m_snapshot_chainstate.reset(); - m_active_chainstate = nullptr; - m_snapshot_validated = false; -} - void ChainstateManager::MaybeRebalanceCaches() { AssertLockHeld(::cs_main); @@ -5095,3 +5211,15 @@ void ChainstateManager::MaybeRebalanceCaches() } } } + +ChainstateManager::~ChainstateManager() +{ + LOCK(::cs_main); + + // TODO: The version bits cache and warning cache should probably become + // non-globals + g_versionbitscache.Clear(); + for (auto& i : warningcache) { + i.clear(); + } +} diff --git a/src/validation.h b/src/validation.h index 7766d77a88..42e41502f9 100644 --- a/src/validation.h +++ b/src/validation.h @@ -131,14 +131,9 @@ extern uint256 hashAssumeValid; /** Minimum work we will assume exists on some valid chain. */ extern arith_uint256 nMinimumChainWork; -/** Best header we've seen so far (used for getheaders queries' starting points). */ -extern CBlockIndex *pindexBestHeader; - /** Documentation for argument 'checklevel'. */ extern const std::vector<std::string> CHECKLEVEL_DOC; -/** Unload database information */ -void UnloadBlockIndex(CTxMemPool* mempool, ChainstateManager& chainman); /** Run instances of script checking worker threads */ void StartScriptCheckWorkerThreads(int threads_num); /** Stop all of the script checking worker threads */ @@ -234,11 +229,19 @@ struct PackageMempoolAcceptResult * was a package-wide error (see result in m_state), m_tx_results will be empty. */ std::map<const uint256, const MempoolAcceptResult> m_tx_results; + /** Package feerate, defined as the aggregated modified fees divided by the total virtual size + * of all transactions in the package. May be unavailable if some inputs were not available or + * a transaction failure caused validation to terminate early. */ + std::optional<CFeeRate> m_package_feerate; explicit PackageMempoolAcceptResult(PackageValidationState state, std::map<const uint256, const MempoolAcceptResult>&& results) : m_state{state}, m_tx_results(std::move(results)) {} + explicit PackageMempoolAcceptResult(PackageValidationState state, CFeeRate feerate, + std::map<const uint256, const MempoolAcceptResult>&& results) + : m_state{state}, m_tx_results(std::move(results)), m_package_feerate{feerate} {} + /** Constructor to create a PackageMempoolAcceptResult from a single MempoolAcceptResult */ explicit PackageMempoolAcceptResult(const uint256& wtxid, const MempoolAcceptResult& result) : m_tx_results{ {wtxid, result} } {} @@ -273,16 +276,12 @@ PackageMempoolAcceptResult ProcessNewPackage(CChainState& active_chainstate, CTx const Package& txns, bool test_accept) EXCLUSIVE_LOCKS_REQUIRED(cs_main); -/** Transaction validation functions */ +/* Transaction policy functions */ /** * Check if transaction will be final in the next block to be created. - * - * Calls IsFinalTx() with current block height and appropriate block time. - * - * See consensus/consensus.h for flag definitions. */ -bool CheckFinalTx(const CBlockIndex* active_chain_tip, const CTransaction &tx, int flags = -1) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +bool CheckFinalTxAtTip(const CBlockIndex* active_chain_tip, const CTransaction& tx) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** * Check if transaction will be BIP68 final in the next block to be created on top of tip. @@ -299,14 +298,11 @@ bool CheckFinalTx(const CBlockIndex* active_chain_tip, const CTransaction &tx, i * Optionally stores in LockPoints the resulting height and time calculated and the hash * of the block needed for calculation or skips the calculation and uses the LockPoints * passed in for evaluation. - * The LockPoints should not be considered valid if CheckSequenceLocks returns false. - * - * See consensus/consensus.h for flag definitions. + * The LockPoints should not be considered valid if CheckSequenceLocksAtTip returns false. */ -bool CheckSequenceLocks(CBlockIndex* tip, +bool CheckSequenceLocksAtTip(CBlockIndex* tip, const CCoinsView& coins_view, const CTransaction& tx, - int flags, LockPoints* lp = nullptr, bool useExistingLockPoints = false); @@ -332,7 +328,8 @@ public: bool operator()(); - void swap(CScriptCheck &check) { + void swap(CScriptCheck& check) noexcept + { std::swap(ptxTo, check.ptxTo); std::swap(m_tx_out, check.m_tx_out); std::swap(nIn, check.nIn); @@ -427,7 +424,7 @@ public: //! state to disk, which should not be done until the health of the database is verified. //! //! All arguments forwarded onto CCoinsViewDB. - CoinsViews(std::string ldb_name, size_t cache_size_bytes, bool in_memory, bool should_wipe); + CoinsViews(fs::path ldb_name, size_t cache_size_bytes, bool in_memory, bool should_wipe); //! Initialize the CCoinsViewCache member. void InitCache() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); @@ -521,7 +518,7 @@ public: size_t cache_size_bytes, bool in_memory, bool should_wipe, - std::string leveldb_name = "chainstate"); + fs::path leveldb_name = "chainstate"); //! Initialize the in-memory coins cache (to be done after the health of the on-disk database //! is verified). @@ -693,7 +690,7 @@ public: bool IsInitialBlockDownload() const; /** Find the last common block of this chain and a locator. */ - CBlockIndex* FindForkInGlobalIndex(const CBlockLocator& locator) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); + const CBlockIndex* FindForkInGlobalIndex(const CBlockLocator& locator) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** * Make various assertions about the state of the block index. @@ -833,10 +830,9 @@ private: //! If true, the assumed-valid chainstate has been fully validated //! by the background validation chainstate. - bool m_snapshot_validated{false}; + bool m_snapshot_validated GUARDED_BY(::cs_main){false}; - CBlockIndex* m_best_invalid; - friend bool node::BlockManager::LoadBlockIndex(const Consensus::Params&, ChainstateManager&); + CBlockIndex* m_best_invalid GUARDED_BY(::cs_main){nullptr}; //! Internal helper for ActivateSnapshot(). [[nodiscard]] bool PopulateAndValidateSnapshot( @@ -882,6 +878,9 @@ public: */ std::set<CBlockIndex*> m_failed_blocks; + /** Best header we've seen so far (used for getheaders queries' starting points). */ + CBlockIndex* m_best_header = nullptr; + //! The total number of bytes available for us to use across all in-memory //! coins caches. This will be split somehow across chainstates. int64_t m_total_coinstip_cache{0}; @@ -940,7 +939,7 @@ public: std::optional<uint256> SnapshotBlockhash() const; //! Is there a snapshot in use and has it been fully validated? - bool IsSnapshotValidated() const { return m_snapshot_validated; } + bool IsSnapshotValidated() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { return m_snapshot_validated; } /** * Process an incoming block. This only returns after the best known valid @@ -988,21 +987,11 @@ public: //! Load the block tree and coins database from disk, initializing state if we're running with -reindex bool LoadBlockIndex() EXCLUSIVE_LOCKS_REQUIRED(cs_main); - //! Unload block index and chain data before shutdown. - void Unload() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - - //! Clear (deconstruct) chainstate data. - void Reset(); - //! Check to see if caches are out of balance and if so, call //! ResizeCoinsCaches() as needed. void MaybeRebalanceCaches() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - ~ChainstateManager() { - LOCK(::cs_main); - UnloadBlockIndex(/* mempool */ nullptr, *this); - Reset(); - } + ~ChainstateManager(); }; using FopenFn = std::function<FILE*(const fs::path&, const char*)>; diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index 1e07ff23ae..edc4633c01 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -40,7 +40,7 @@ public: // our own queue here :( SingleThreadedSchedulerClient m_schedulerClient; - explicit MainSignalsInstance(CScheduler *pscheduler) : m_schedulerClient(pscheduler) {} + explicit MainSignalsInstance(CScheduler& scheduler LIFETIMEBOUND) : m_schedulerClient(scheduler) {} void Register(std::shared_ptr<CValidationInterface> callbacks) { @@ -92,7 +92,7 @@ static CMainSignals g_signals; void CMainSignals::RegisterBackgroundSignalScheduler(CScheduler& scheduler) { assert(!m_internals); - m_internals.reset(new MainSignalsInstance(&scheduler)); + m_internals = std::make_unique<MainSignalsInstance>(scheduler); } void CMainSignals::UnregisterBackgroundSignalScheduler() diff --git a/src/validationinterface.h b/src/validationinterface.h index 7c3ce00fbc..ac62f8b467 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -174,6 +174,7 @@ protected: * has been received and connected to the headers tree, though not validated yet */ virtual void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& block) {}; friend class CMainSignals; + friend class ValidationInterfaceTest; }; struct MainSignalsInstance; diff --git a/src/versionbits.cpp b/src/versionbits.cpp index 7a297c2bbb..dc85655028 100644 --- a/src/versionbits.cpp +++ b/src/versionbits.cpp @@ -2,8 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <versionbits.h> #include <consensus/params.h> +#include <util/check.h> +#include <versionbits.h> ThresholdState AbstractThresholdConditionChecker::GetStateFor(const CBlockIndex* pindexPrev, const Consensus::Params& params, ThresholdConditionCache& cache) const { @@ -158,7 +159,7 @@ int AbstractThresholdConditionChecker::GetStateSinceHeightFor(const CBlockIndex* // if we are computing for the last block of a period, then pindexPrev points to the second to last block of the period, and // if we are computing for the first block of a period, then pindexPrev points to the last block of the previous period. // The parent of the genesis block is represented by nullptr. - pindexPrev = pindexPrev->GetAncestor(pindexPrev->nHeight - ((pindexPrev->nHeight + 1) % nPeriod)); + pindexPrev = Assert(pindexPrev->GetAncestor(pindexPrev->nHeight - ((pindexPrev->nHeight + 1) % nPeriod))); const CBlockIndex* previousPeriodParent = pindexPrev->GetAncestor(pindexPrev->nHeight - nPeriod); diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index 49f0abf9e7..1aa0339445 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -60,12 +60,12 @@ bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId& rhs) const * erases the weak pointer from the g_dbenvs map. * @post A new BerkeleyEnvironment weak pointer is inserted into g_dbenvs if the directory path key was not already in the map. */ -std::shared_ptr<BerkeleyEnvironment> GetBerkeleyEnv(const fs::path& env_directory) +std::shared_ptr<BerkeleyEnvironment> GetBerkeleyEnv(const fs::path& env_directory, bool use_shared_memory) { LOCK(cs_db); auto inserted = g_dbenvs.emplace(fs::PathToString(env_directory), std::weak_ptr<BerkeleyEnvironment>()); if (inserted.second) { - auto env = std::make_shared<BerkeleyEnvironment>(env_directory); + auto env = std::make_shared<BerkeleyEnvironment>(env_directory, use_shared_memory); inserted.first->second = env; return env; } @@ -113,7 +113,7 @@ void BerkeleyEnvironment::Reset() fMockDb = false; } -BerkeleyEnvironment::BerkeleyEnvironment(const fs::path& dir_path) : strPath(fs::PathToString(dir_path)) +BerkeleyEnvironment::BerkeleyEnvironment(const fs::path& dir_path, bool use_shared_memory) : strPath(fs::PathToString(dir_path)), m_use_shared_memory(use_shared_memory) { Reset(); } @@ -145,8 +145,9 @@ bool BerkeleyEnvironment::Open(bilingual_str& err) LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", fs::PathToString(pathLogDir), fs::PathToString(pathErrorFile)); unsigned int nEnvFlags = 0; - if (gArgs.GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB)) + if (!m_use_shared_memory) { nEnvFlags |= DB_PRIVATE; + } dbenv->set_lg_dir(fs::PathToString(pathLogDir).c_str()); dbenv->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet @@ -188,7 +189,7 @@ bool BerkeleyEnvironment::Open(bilingual_str& err) } //! Construct an in-memory mock Berkeley environment for testing -BerkeleyEnvironment::BerkeleyEnvironment() +BerkeleyEnvironment::BerkeleyEnvironment() : m_use_shared_memory(false) { Reset(); @@ -260,7 +261,7 @@ BerkeleyBatch::SafeDbt::operator Dbt*() bool BerkeleyDatabase::Verify(bilingual_str& errorStr) { fs::path walletDir = env->Directory(); - fs::path file_path = walletDir / strFile; + fs::path file_path = walletDir / m_filename; LogPrintf("Using BerkeleyDB version %s\n", BerkeleyDatabaseVersion()); LogPrintf("Using wallet %s\n", fs::PathToString(file_path)); @@ -274,6 +275,7 @@ bool BerkeleyDatabase::Verify(bilingual_str& errorStr) assert(m_refcount == 0); Db db(env->dbenv.get(), 0); + const std::string strFile = fs::PathToString(m_filename); int result = db.verify(strFile.c_str(), nullptr, nullptr, 0); if (result != 0) { errorStr = strprintf(_("%s corrupt. Try using the wallet tool bitcoin-wallet to salvage or restoring a backup."), fs::quoted(fs::PathToString(file_path))); @@ -296,11 +298,11 @@ BerkeleyDatabase::~BerkeleyDatabase() { if (env) { LOCK(cs_db); - env->CloseDb(strFile); + env->CloseDb(m_filename); assert(!m_db); - size_t erased = env->m_databases.erase(strFile); + size_t erased = env->m_databases.erase(m_filename); assert(erased == 1); - env->m_fileids.erase(strFile); + env->m_fileids.erase(fs::PathToString(m_filename)); } } @@ -312,7 +314,7 @@ BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const bool read_only, b fFlushOnClose = fFlushOnCloseIn; env = database.env.get(); pdb = database.m_db.get(); - strFile = database.strFile; + strFile = fs::PathToString(database.m_filename); if (!Exists(std::string("version"))) { bool fTmp = fReadOnly; fReadOnly = false; @@ -334,6 +336,7 @@ void BerkeleyDatabase::Open() if (m_db == nullptr) { int ret; std::unique_ptr<Db> pdb_temp = std::make_unique<Db>(env->dbenv.get(), 0); + const std::string strFile = fs::PathToString(m_filename); bool fMockDb = env->IsMock(); if (fMockDb) { @@ -377,7 +380,7 @@ void BerkeleyBatch::Flush() nMinutes = 1; if (env) { // env is nullptr for dummy databases (i.e. in tests). Don't actually flush if env is nullptr so we don't segfault - env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetIntArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); + env->dbenv->txn_checkpoint(nMinutes ? m_database.m_max_log_mb * 1024 : 0, nMinutes, 0); } } @@ -406,11 +409,11 @@ void BerkeleyBatch::Close() Flush(); } -void BerkeleyEnvironment::CloseDb(const std::string& strFile) +void BerkeleyEnvironment::CloseDb(const fs::path& filename) { { LOCK(cs_db); - auto it = m_databases.find(strFile); + auto it = m_databases.find(filename); assert(it != m_databases.end()); BerkeleyDatabase& database = it->second.get(); if (database.m_db) { @@ -433,12 +436,12 @@ void BerkeleyEnvironment::ReloadDbEnv() return true; }); - std::vector<std::string> filenames; + std::vector<fs::path> filenames; for (auto it : m_databases) { filenames.push_back(it.first); } // Close the individual Db's - for (const std::string& filename : filenames) { + for (const fs::path& filename : filenames) { CloseDb(filename); } // Reset the environment @@ -453,9 +456,10 @@ bool BerkeleyDatabase::Rewrite(const char* pszSkip) while (true) { { LOCK(cs_db); + const std::string strFile = fs::PathToString(m_filename); if (m_refcount <= 0) { // Flush log data to the dat file - env->CloseDb(strFile); + env->CloseDb(m_filename); env->CheckpointLSN(strFile); m_refcount = -1; @@ -507,7 +511,7 @@ bool BerkeleyDatabase::Rewrite(const char* pszSkip) } if (fSuccess) { db.Close(); - env->CloseDb(strFile); + env->CloseDb(m_filename); if (pdbCopy->close(0)) fSuccess = false; } else { @@ -543,13 +547,14 @@ void BerkeleyEnvironment::Flush(bool fShutdown) LOCK(cs_db); bool no_dbs_accessed = true; for (auto& db_it : m_databases) { - std::string strFile = db_it.first; + const fs::path& filename = db_it.first; int nRefCount = db_it.second.get().m_refcount; if (nRefCount < 0) continue; + const std::string strFile = fs::PathToString(filename); LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount); if (nRefCount == 0) { // Move log data to the dat file - CloseDb(strFile); + CloseDb(filename); LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile); dbenv->txn_checkpoint(0, 0, 0); LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s detach\n", strFile); @@ -589,11 +594,12 @@ bool BerkeleyDatabase::PeriodicFlush() // Don't flush if there haven't been any batch writes for this database. if (m_refcount < 0) return false; + const std::string strFile = fs::PathToString(m_filename); LogPrint(BCLog::WALLETDB, "Flushing %s\n", strFile); int64_t nStart = GetTimeMillis(); // Flush wallet file so it's self contained - env->CloseDb(strFile); + env->CloseDb(m_filename); env->CheckpointLSN(strFile); m_refcount = -1; @@ -604,6 +610,7 @@ bool BerkeleyDatabase::PeriodicFlush() bool BerkeleyDatabase::Backup(const std::string& strDest) const { + const std::string strFile = fs::PathToString(m_filename); while (true) { { @@ -611,14 +618,14 @@ bool BerkeleyDatabase::Backup(const std::string& strDest) const if (m_refcount <= 0) { // Flush log data to the dat file - env->CloseDb(strFile); + env->CloseDb(m_filename); env->CheckpointLSN(strFile); // Copy wallet file - fs::path pathSrc = env->Directory() / strFile; + fs::path pathSrc = env->Directory() / m_filename; fs::path pathDest(fs::PathFromString(strDest)); if (fs::is_directory(pathDest)) - pathDest /= fs::PathFromString(strFile); + pathDest /= m_filename; try { if (fs::exists(pathDest) && fs::equivalent(pathSrc, pathDest)) { @@ -682,10 +689,10 @@ bool BerkeleyBatch::ReadAtCursor(CDataStream& ssKey, CDataStream& ssValue, bool& // Convert to streams ssKey.SetType(SER_DISK); ssKey.clear(); - ssKey.write({BytePtr(datKey.get_data()), datKey.get_size()}); + ssKey.write({AsBytePtr(datKey.get_data()), datKey.get_size()}); ssValue.SetType(SER_DISK); ssValue.clear(); - ssValue.write({BytePtr(datValue.get_data()), datValue.get_size()}); + ssValue.write({AsBytePtr(datValue.get_data()), datValue.get_size()}); return true; } @@ -757,7 +764,7 @@ bool BerkeleyBatch::ReadKey(CDataStream&& key, CDataStream& value) SafeDbt datValue; int ret = pdb->get(activeTxn, datKey, datValue, 0); if (ret == 0 && datValue.get_data() != nullptr) { - value.write({BytePtr(datValue.get_data()), datValue.get_size()}); + value.write({AsBytePtr(datValue.get_data()), datValue.get_size()}); return true; } return false; @@ -830,14 +837,14 @@ std::unique_ptr<BerkeleyDatabase> MakeBerkeleyDatabase(const fs::path& path, con std::unique_ptr<BerkeleyDatabase> db; { LOCK(cs_db); // Lock env.m_databases until insert in BerkeleyDatabase constructor - std::string data_filename = fs::PathToString(data_file.filename()); - std::shared_ptr<BerkeleyEnvironment> env = GetBerkeleyEnv(data_file.parent_path()); + fs::path data_filename = data_file.filename(); + std::shared_ptr<BerkeleyEnvironment> env = GetBerkeleyEnv(data_file.parent_path(), options.use_shared_memory); if (env->m_databases.count(data_filename)) { error = Untranslated(strprintf("Refusing to load database. Data file '%s' is already loaded.", fs::PathToString(env->Directory() / data_filename))); status = DatabaseStatus::FAILED_ALREADY_LOADED; return nullptr; } - db = std::make_unique<BerkeleyDatabase>(std::move(env), std::move(data_filename)); + db = std::make_unique<BerkeleyDatabase>(std::move(env), std::move(data_filename), options); } if (options.verify && !db->Verify(error)) { diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h index b924890d81..1c99e1f9af 100644 --- a/src/wallet/bdb.h +++ b/src/wallet/bdb.h @@ -32,8 +32,6 @@ struct bilingual_str; namespace wallet { -static const unsigned int DEFAULT_WALLET_DBLOGSIZE = 100; -static const bool DEFAULT_WALLET_PRIVDB = true; struct WalletDatabaseFileId { u_int8_t value[DB_FILE_ID_LEN]; @@ -53,11 +51,12 @@ private: public: std::unique_ptr<DbEnv> dbenv; - std::map<std::string, std::reference_wrapper<BerkeleyDatabase>> m_databases; + std::map<fs::path, std::reference_wrapper<BerkeleyDatabase>> m_databases; std::unordered_map<std::string, WalletDatabaseFileId> m_fileids; std::condition_variable_any m_db_in_use; + bool m_use_shared_memory; - explicit BerkeleyEnvironment(const fs::path& env_directory); + explicit BerkeleyEnvironment(const fs::path& env_directory, bool use_shared_memory); BerkeleyEnvironment(); ~BerkeleyEnvironment(); void Reset(); @@ -71,7 +70,7 @@ public: void Flush(bool fShutdown); void CheckpointLSN(const std::string& strFile); - void CloseDb(const std::string& strFile); + void CloseDb(const fs::path& filename); void ReloadDbEnv(); DbTxn* TxnBegin(int flags = DB_TXN_WRITE_NOSYNC) @@ -85,7 +84,7 @@ public: }; /** Get BerkeleyEnvironment given a directory path. */ -std::shared_ptr<BerkeleyEnvironment> GetBerkeleyEnv(const fs::path& env_directory); +std::shared_ptr<BerkeleyEnvironment> GetBerkeleyEnv(const fs::path& env_directory, bool use_shared_memory); class BerkeleyBatch; @@ -98,10 +97,10 @@ public: BerkeleyDatabase() = delete; /** Create DB handle to real database */ - BerkeleyDatabase(std::shared_ptr<BerkeleyEnvironment> env, std::string filename) : - WalletDatabase(), env(std::move(env)), strFile(std::move(filename)) + BerkeleyDatabase(std::shared_ptr<BerkeleyEnvironment> env, fs::path filename, const DatabaseOptions& options) : + WalletDatabase(), env(std::move(env)), m_filename(std::move(filename)), m_max_log_mb(options.max_log_mb) { - auto inserted = this->env->m_databases.emplace(strFile, std::ref(*this)); + auto inserted = this->env->m_databases.emplace(m_filename, std::ref(*this)); assert(inserted.second); } @@ -142,7 +141,7 @@ public: bool Verify(bilingual_str& error); /** Return path to main database filename */ - std::string Filename() override { return fs::PathToString(env->Directory() / strFile); } + std::string Filename() override { return fs::PathToString(env->Directory() / m_filename); } std::string Format() override { return "bdb"; } /** @@ -159,7 +158,8 @@ public: /** Database pointer. This is initialized lazily and reset during flushes, so it can be null. */ std::unique_ptr<Db> m_db; - std::string strFile; + fs::path m_filename; + int64_t m_max_log_mb; /** Make a BerkeleyBatch connected to this database */ std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) override; diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 23faad027f..0b236e2e48 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -50,8 +50,8 @@ struct { * The Branch and Bound algorithm is described in detail in Murch's Master Thesis: * https://murch.one/wp-content/uploads/2016/11/erhardt2016coinselection.pdf * - * @param const std::vector<CInputCoin>& utxo_pool The set of UTXOs that we are choosing from. - * These UTXOs will be sorted in descending order by effective value and the CInputCoins' + * @param const std::vector<OutputGroup>& utxo_pool The set of UTXO groups that we are choosing from. + * These UTXO groups will be sorted in descending order by effective value and the OutputGroups' * values are their effective values. * @param const CAmount& selection_target This is the value that we want to select. It is the lower * bound of the range. @@ -64,16 +64,15 @@ static const size_t TOTAL_TRIES = 100000; std::optional<SelectionResult> SelectCoinsBnB(std::vector<OutputGroup>& utxo_pool, const CAmount& selection_target, const CAmount& cost_of_change) { - SelectionResult result(selection_target); + SelectionResult result(selection_target, SelectionAlgorithm::BNB); CAmount curr_value = 0; - - std::vector<bool> curr_selection; // select the utxo at this index - curr_selection.reserve(utxo_pool.size()); + std::vector<size_t> curr_selection; // selected utxo indexes // Calculate curr_available_value CAmount curr_available_value = 0; for (const OutputGroup& utxo : utxo_pool) { - // Assert that this utxo is not negative. It should never be negative, effective value calculation should have removed it + // Assert that this utxo is not negative. It should never be negative, + // effective value calculation should have removed it assert(utxo.GetSelectionAmount() > 0); curr_available_value += utxo.GetSelectionAmount(); } @@ -85,15 +84,15 @@ std::optional<SelectionResult> SelectCoinsBnB(std::vector<OutputGroup>& utxo_poo std::sort(utxo_pool.begin(), utxo_pool.end(), descending); CAmount curr_waste = 0; - std::vector<bool> best_selection; + std::vector<size_t> best_selection; CAmount best_waste = MAX_MONEY; // Depth First search loop for choosing the UTXOs - for (size_t i = 0; i < TOTAL_TRIES; ++i) { + for (size_t curr_try = 0, utxo_pool_index = 0; curr_try < TOTAL_TRIES; ++curr_try, ++utxo_pool_index) { // Conditions for starting a backtrack bool backtrack = false; - if (curr_value + curr_available_value < selection_target || // Cannot possibly reach target with the amount remaining in the curr_available_value. - curr_value > selection_target + cost_of_change || // Selected value is out of range, go back and try other branch + if (curr_value + curr_available_value < selection_target || // Cannot possibly reach target with the amount remaining in the curr_available_value. + curr_value > selection_target + cost_of_change || // Selected value is out of range, go back and try other branch (curr_waste > best_waste && (utxo_pool.at(0).fee - utxo_pool.at(0).long_term_fee) > 0)) { // Don't select things which we know will be more wasteful if the waste is increasing backtrack = true; } else if (curr_value >= selection_target) { // Selected value is within range @@ -104,7 +103,6 @@ std::optional<SelectionResult> SelectCoinsBnB(std::vector<OutputGroup>& utxo_poo // explore any more UTXOs to avoid burning money like that. if (curr_waste <= best_waste) { best_selection = curr_selection; - best_selection.resize(utxo_pool.size()); best_waste = curr_waste; if (best_waste == 0) { break; @@ -114,38 +112,38 @@ std::optional<SelectionResult> SelectCoinsBnB(std::vector<OutputGroup>& utxo_poo backtrack = true; } - // Backtracking, moving backwards - if (backtrack) { - // Walk backwards to find the last included UTXO that still needs to have its omission branch traversed. - while (!curr_selection.empty() && !curr_selection.back()) { - curr_selection.pop_back(); - curr_available_value += utxo_pool.at(curr_selection.size()).GetSelectionAmount(); - } - + if (backtrack) { // Backtracking, moving backwards if (curr_selection.empty()) { // We have walked back to the first utxo and no branch is untraversed. All solutions searched break; } + // Add omitted UTXOs back to lookahead before traversing the omission branch of last included UTXO. + for (--utxo_pool_index; utxo_pool_index > curr_selection.back(); --utxo_pool_index) { + curr_available_value += utxo_pool.at(utxo_pool_index).GetSelectionAmount(); + } + // Output was included on previous iterations, try excluding now. - curr_selection.back() = false; - OutputGroup& utxo = utxo_pool.at(curr_selection.size() - 1); + assert(utxo_pool_index == curr_selection.back()); + OutputGroup& utxo = utxo_pool.at(utxo_pool_index); curr_value -= utxo.GetSelectionAmount(); curr_waste -= utxo.fee - utxo.long_term_fee; + curr_selection.pop_back(); } else { // Moving forwards, continuing down this branch - OutputGroup& utxo = utxo_pool.at(curr_selection.size()); + OutputGroup& utxo = utxo_pool.at(utxo_pool_index); // Remove this utxo from the curr_available_value utxo amount curr_available_value -= utxo.GetSelectionAmount(); - // Avoid searching a branch if the previous UTXO has the same value and same waste and was excluded. Since the ratio of fee to - // long term fee is the same, we only need to check if one of those values match in order to know that the waste is the same. - if (!curr_selection.empty() && !curr_selection.back() && - utxo.GetSelectionAmount() == utxo_pool.at(curr_selection.size() - 1).GetSelectionAmount() && - utxo.fee == utxo_pool.at(curr_selection.size() - 1).fee) { - curr_selection.push_back(false); - } else { + if (curr_selection.empty() || + // The previous index is included and therefore not relevant for exclusion shortcut + (utxo_pool_index - 1) == curr_selection.back() || + // Avoid searching a branch if the previous UTXO has the same value and same waste and was excluded. + // Since the ratio of fee to long term fee is the same, we only need to check if one of those values match in order to know that the waste is the same. + utxo.GetSelectionAmount() != utxo_pool.at(utxo_pool_index - 1).GetSelectionAmount() || + utxo.fee != utxo_pool.at(utxo_pool_index - 1).fee) + { // Inclusion branch first (Largest First Exploration) - curr_selection.push_back(true); + curr_selection.push_back(utxo_pool_index); curr_value += utxo.GetSelectionAmount(); curr_waste += utxo.fee - utxo.long_term_fee; } @@ -158,23 +156,23 @@ std::optional<SelectionResult> SelectCoinsBnB(std::vector<OutputGroup>& utxo_poo } // Set output set - for (size_t i = 0; i < best_selection.size(); ++i) { - if (best_selection.at(i)) { - result.AddInput(utxo_pool.at(i)); - } + for (const size_t& i : best_selection) { + result.AddInput(utxo_pool.at(i)); } + result.ComputeAndSetWaste(CAmount{0}); + assert(best_waste == result.GetWaste()); return result; } -std::optional<SelectionResult> SelectCoinsSRD(const std::vector<OutputGroup>& utxo_pool, CAmount target_value) +std::optional<SelectionResult> SelectCoinsSRD(const std::vector<OutputGroup>& utxo_pool, CAmount target_value, FastRandomContext& rng) { - SelectionResult result(target_value); + SelectionResult result(target_value, SelectionAlgorithm::SRD); std::vector<size_t> indexes; indexes.resize(utxo_pool.size()); std::iota(indexes.begin(), indexes.end(), 0); - Shuffle(indexes.begin(), indexes.end(), FastRandomContext()); + Shuffle(indexes.begin(), indexes.end(), rng); CAmount selected_eff_value = 0; for (const size_t i : indexes) { @@ -189,16 +187,27 @@ std::optional<SelectionResult> SelectCoinsSRD(const std::vector<OutputGroup>& ut return std::nullopt; } -static void ApproximateBestSubset(const std::vector<OutputGroup>& groups, const CAmount& nTotalLower, const CAmount& nTargetValue, +/** Find a subset of the OutputGroups that is at least as large as, but as close as possible to, the + * target amount; solve subset sum. + * param@[in] groups OutputGroups to choose from, sorted by value in descending order. + * param@[in] nTotalLower Total (effective) value of the UTXOs in groups. + * param@[in] nTargetValue Subset sum target, not including change. + * param@[out] vfBest Boolean vector representing the subset chosen that is closest to + * nTargetValue, with indices corresponding to groups. If the ith + * entry is true, that means the ith group in groups was selected. + * param@[out] nBest Total amount of subset chosen that is closest to nTargetValue. + * param@[in] iterations Maximum number of tries. + */ +static void ApproximateBestSubset(FastRandomContext& insecure_rand, const std::vector<OutputGroup>& groups, + const CAmount& nTotalLower, const CAmount& nTargetValue, std::vector<char>& vfBest, CAmount& nBest, int iterations = 1000) { std::vector<char> vfIncluded; + // Worst case "best" approximation is just all of the groups. vfBest.assign(groups.size(), true); nBest = nTotalLower; - FastRandomContext insecure_rand; - for (int nRep = 0; nRep < iterations && nBest != nTargetValue; nRep++) { vfIncluded.assign(groups.size(), false); @@ -221,6 +230,8 @@ static void ApproximateBestSubset(const std::vector<OutputGroup>& groups, const if (nTotal >= nTargetValue) { fReachedTarget = true; + // If the total is between nTargetValue and nBest, it's our new best + // approximation. if (nTotal < nBest) { nBest = nTotal; @@ -235,22 +246,25 @@ static void ApproximateBestSubset(const std::vector<OutputGroup>& groups, const } } -std::optional<SelectionResult> KnapsackSolver(std::vector<OutputGroup>& groups, const CAmount& nTargetValue) +std::optional<SelectionResult> KnapsackSolver(std::vector<OutputGroup>& groups, const CAmount& nTargetValue, + CAmount change_target, FastRandomContext& rng) { - SelectionResult result(nTargetValue); + SelectionResult result(nTargetValue, SelectionAlgorithm::KNAPSACK); // List of values less than target std::optional<OutputGroup> lowest_larger; + // Groups with selection amount smaller than the target and any change we might produce. + // Don't include groups larger than this, because they will only cause us to overshoot. std::vector<OutputGroup> applicable_groups; CAmount nTotalLower = 0; - Shuffle(groups.begin(), groups.end(), FastRandomContext()); + Shuffle(groups.begin(), groups.end(), rng); for (const OutputGroup& group : groups) { if (group.GetSelectionAmount() == nTargetValue) { result.AddInput(group); return result; - } else if (group.GetSelectionAmount() < nTargetValue + MIN_CHANGE) { + } else if (group.GetSelectionAmount() < nTargetValue + change_target) { applicable_groups.push_back(group); nTotalLower += group.GetSelectionAmount(); } else if (!lowest_larger || group.GetSelectionAmount() < lowest_larger->GetSelectionAmount()) { @@ -276,15 +290,15 @@ std::optional<SelectionResult> KnapsackSolver(std::vector<OutputGroup>& groups, std::vector<char> vfBest; CAmount nBest; - ApproximateBestSubset(applicable_groups, nTotalLower, nTargetValue, vfBest, nBest); - if (nBest != nTargetValue && nTotalLower >= nTargetValue + MIN_CHANGE) { - ApproximateBestSubset(applicable_groups, nTotalLower, nTargetValue + MIN_CHANGE, vfBest, nBest); + ApproximateBestSubset(rng, applicable_groups, nTotalLower, nTargetValue, vfBest, nBest); + if (nBest != nTargetValue && nTotalLower >= nTargetValue + change_target) { + ApproximateBestSubset(rng, applicable_groups, nTotalLower, nTargetValue + change_target, vfBest, nBest); } // If we have a bigger coin and (either the stochastic approximation didn't find a good solution, // or the next bigger coin is closer), return the bigger coin if (lowest_larger && - ((nBest != nTargetValue && nBest < nTargetValue + MIN_CHANGE) || lowest_larger->GetSelectionAmount() <= nBest)) { + ((nBest != nTargetValue && nBest < nTargetValue + change_target) || lowest_larger->GetSelectionAmount() <= nBest)) { result.AddInput(*lowest_larger); } else { for (unsigned int i = 0; i < applicable_groups.size(); i++) { @@ -313,29 +327,29 @@ std::optional<SelectionResult> KnapsackSolver(std::vector<OutputGroup>& groups, ******************************************************************************/ -void OutputGroup::Insert(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants, bool positive_only) { +void OutputGroup::Insert(const COutput& output, size_t ancestors, size_t descendants, bool positive_only) { // Compute the effective value first - const CAmount coin_fee = output.m_input_bytes < 0 ? 0 : m_effective_feerate.GetFee(output.m_input_bytes); + const CAmount coin_fee = output.input_bytes < 0 ? 0 : m_effective_feerate.GetFee(output.input_bytes); const CAmount ev = output.txout.nValue - coin_fee; // Filter for positive only here before adding the coin if (positive_only && ev <= 0) return; m_outputs.push_back(output); - CInputCoin& coin = m_outputs.back(); + COutput& coin = m_outputs.back(); - coin.m_fee = coin_fee; - fee += coin.m_fee; + coin.fee = coin_fee; + fee += coin.fee; - coin.m_long_term_fee = coin.m_input_bytes < 0 ? 0 : m_long_term_feerate.GetFee(coin.m_input_bytes); - long_term_fee += coin.m_long_term_fee; + coin.long_term_fee = coin.input_bytes < 0 ? 0 : m_long_term_feerate.GetFee(coin.input_bytes); + long_term_fee += coin.long_term_fee; coin.effective_value = ev; effective_value += coin.effective_value; - m_from_me &= from_me; - m_value += output.txout.nValue; - m_depth = std::min(m_depth, depth); + m_from_me &= coin.from_me; + m_value += coin.txout.nValue; + m_depth = std::min(m_depth, coin.depth); // ancestors here express the number of ancestors the new coin will end up having, which is // the sum, rather than the max; this will overestimate in the cases where multiple inputs // have common ancestors @@ -357,7 +371,7 @@ CAmount OutputGroup::GetSelectionAmount() const return m_subtract_fee_outputs ? m_value : effective_value; } -CAmount GetSelectionWaste(const std::set<CInputCoin>& inputs, CAmount change_cost, CAmount target, bool use_effective_value) +CAmount GetSelectionWaste(const std::set<COutput>& inputs, CAmount change_cost, CAmount target, bool use_effective_value) { // This function should not be called with empty inputs as that would mean the selection failed assert(!inputs.empty()); @@ -365,8 +379,8 @@ CAmount GetSelectionWaste(const std::set<CInputCoin>& inputs, CAmount change_cos // Always consider the cost of spending an input now vs in the future. CAmount waste = 0; CAmount selected_effective_value = 0; - for (const CInputCoin& coin : inputs) { - waste += coin.m_fee - coin.m_long_term_fee; + for (const COutput& coin : inputs) { + waste += coin.fee - coin.long_term_fee; selected_effective_value += use_effective_value ? coin.effective_value : coin.txout.nValue; } @@ -384,6 +398,17 @@ CAmount GetSelectionWaste(const std::set<CInputCoin>& inputs, CAmount change_cos return waste; } +CAmount GenerateChangeTarget(CAmount payment_value, FastRandomContext& rng) +{ + if (payment_value <= CHANGE_LOWER / 2) { + return CHANGE_LOWER; + } else { + // random value between 50ksat and min (payment_value * 2, 1milsat) + const auto upper_bound = std::min(payment_value * 2, CHANGE_UPPER); + return rng.randrange(upper_bound - CHANGE_LOWER) + CHANGE_LOWER; + } +} + void SelectionResult::ComputeAndSetWaste(CAmount change_cost) { m_waste = GetSelectionWaste(m_selected_inputs, change_cost, m_target, m_use_effective); @@ -411,14 +436,14 @@ void SelectionResult::AddInput(const OutputGroup& group) m_use_effective = !group.m_subtract_fee_outputs; } -const std::set<CInputCoin>& SelectionResult::GetInputSet() const +const std::set<COutput>& SelectionResult::GetInputSet() const { return m_selected_inputs; } -std::vector<CInputCoin> SelectionResult::GetShuffledInputVector() const +std::vector<COutput> SelectionResult::GetShuffledInputVector() const { - std::vector<CInputCoin> coins(m_selected_inputs.begin(), m_selected_inputs.end()); + std::vector<COutput> coins(m_selected_inputs.begin(), m_selected_inputs.end()); Shuffle(coins.begin(), coins.end(), FastRandomContext()); return coins; } @@ -430,4 +455,22 @@ bool SelectionResult::operator<(SelectionResult other) const // As this operator is only used in std::min_element, we want the result that has more inputs when waste are equal. return *m_waste < *other.m_waste || (*m_waste == *other.m_waste && m_selected_inputs.size() > other.m_selected_inputs.size()); } + +std::string COutput::ToString() const +{ + return strprintf("COutput(%s, %d, %d) [%s]", outpoint.hash.ToString(), outpoint.n, depth, FormatMoney(txout.nValue)); +} + +std::string GetAlgorithmName(const SelectionAlgorithm algo) +{ + switch (algo) + { + case SelectionAlgorithm::BNB: return "bnb"; + case SelectionAlgorithm::KNAPSACK: return "knapsack"; + case SelectionAlgorithm::SRD: return "srd"; + case SelectionAlgorithm::MANUAL: return "manual"; + // No default case to allow for compiler to warn + } + assert(false); +} } // namespace wallet diff --git a/src/wallet/coinselection.h b/src/wallet/coinselection.h index 496a026999..25c672eda0 100644 --- a/src/wallet/coinselection.h +++ b/src/wallet/coinselection.h @@ -13,74 +13,93 @@ #include <optional> namespace wallet { -//! target minimum change amount -static constexpr CAmount MIN_CHANGE{COIN / 100}; -//! final minimum change amount after paying for fees -static const CAmount MIN_FINAL_CHANGE = MIN_CHANGE/2; +//! lower bound for randomly-chosen target change amount +static constexpr CAmount CHANGE_LOWER{50000}; +//! upper bound for randomly-chosen target change amount +static constexpr CAmount CHANGE_UPPER{1000000}; /** A UTXO under consideration for use in funding a new transaction. */ -class CInputCoin { -public: - CInputCoin(const CTransactionRef& tx, unsigned int i) - { - if (!tx) - throw std::invalid_argument("tx should not be null"); - if (i >= tx->vout.size()) - throw std::out_of_range("The output index is out of range"); - - outpoint = COutPoint(tx->GetHash(), i); - txout = tx->vout[i]; - effective_value = txout.nValue; - } +struct COutput { + /** The outpoint identifying this UTXO */ + COutPoint outpoint; - CInputCoin(const CTransactionRef& tx, unsigned int i, int input_bytes) : CInputCoin(tx, i) - { - m_input_bytes = input_bytes; - } + /** The output itself */ + CTxOut txout; - CInputCoin(const COutPoint& outpoint_in, const CTxOut& txout_in) - { - outpoint = outpoint_in; - txout = txout_in; - effective_value = txout.nValue; - } + /** + * Depth in block chain. + * If > 0: the tx is on chain and has this many confirmations. + * If = 0: the tx is waiting confirmation. + * If < 0: a conflicting tx is on chain and has this many confirmations. */ + int depth; - CInputCoin(const COutPoint& outpoint_in, const CTxOut& txout_in, int input_bytes) : CInputCoin(outpoint_in, txout_in) - { - m_input_bytes = input_bytes; - } + /** Pre-computed estimated size of this output as a fully-signed input in a transaction. Can be -1 if it could not be calculated */ + int input_bytes; - COutPoint outpoint; - CTxOut txout; + /** Whether we have the private keys to spend this output */ + bool spendable; + + /** Whether we know how to spend this output, ignoring the lack of keys */ + bool solvable; + + /** + * Whether this output is considered safe to spend. Unconfirmed transactions + * from outside keys and unconfirmed replacement transactions are considered + * unsafe and will not be used to fund new spending transactions. + */ + bool safe; + + /** The time of the transaction containing this output as determined by CWalletTx::nTimeSmart */ + int64_t time; + + /** Whether the transaction containing this output is sent from the owning wallet */ + bool from_me; + + /** The output's value minus fees required to spend it. Initialized as the output's absolute value. */ CAmount effective_value; - CAmount m_fee{0}; - CAmount m_long_term_fee{0}; - /** Pre-computed estimated size of this output as a fully-signed input in a transaction. Can be -1 if it could not be calculated */ - int m_input_bytes{-1}; + /** The fee required to spend this output at the transaction's target feerate. */ + CAmount fee{0}; - bool operator<(const CInputCoin& rhs) const { - return outpoint < rhs.outpoint; - } + /** The fee required to spend this output at the consolidation feerate. */ + CAmount long_term_fee{0}; - bool operator!=(const CInputCoin& rhs) const { - return outpoint != rhs.outpoint; - } + COutput(const COutPoint& outpoint, const CTxOut& txout, int depth, int input_bytes, bool spendable, bool solvable, bool safe, int64_t time, bool from_me) + : outpoint{outpoint}, + txout{txout}, + depth{depth}, + input_bytes{input_bytes}, + spendable{spendable}, + solvable{solvable}, + safe{safe}, + time{time}, + from_me{from_me}, + effective_value{txout.nValue} + {} + + std::string ToString() const; - bool operator==(const CInputCoin& rhs) const { - return outpoint == rhs.outpoint; + bool operator<(const COutput& rhs) const + { + return outpoint < rhs.outpoint; } }; /** Parameters for one iteration of Coin Selection. */ -struct CoinSelectionParams -{ +struct CoinSelectionParams { + /** Randomness to use in the context of coin selection. */ + FastRandomContext& rng_fast; /** Size of a change output in bytes, determined by the output type. */ size_t change_output_size = 0; /** Size of the input to spend a change output in virtual bytes. */ size_t change_spend_size = 0; + /** Mininmum change to target in Knapsack solver: select coins to cover the payment and + * at least this value of change. */ + CAmount m_min_change_target{0}; /** Cost of creating the change output. */ CAmount m_change_fee{0}; + /** The pre-determined minimum value to target when funding a change output. */ + CAmount m_change_target{0}; /** Cost of creating the change output + cost of spending the change output in the future. */ CAmount m_cost_of_change{0}; /** The targeted feerate of the transaction being built. */ @@ -100,17 +119,22 @@ struct CoinSelectionParams * reuse. Dust outputs are not eligible to be added to output groups and thus not considered. */ bool m_avoid_partial_spends = false; - CoinSelectionParams(size_t change_output_size, size_t change_spend_size, CFeeRate effective_feerate, - CFeeRate long_term_feerate, CFeeRate discard_feerate, size_t tx_noinputs_size, bool avoid_partial) : - change_output_size(change_output_size), - change_spend_size(change_spend_size), - m_effective_feerate(effective_feerate), - m_long_term_feerate(long_term_feerate), - m_discard_feerate(discard_feerate), - tx_noinputs_size(tx_noinputs_size), - m_avoid_partial_spends(avoid_partial) - {} - CoinSelectionParams() {} + CoinSelectionParams(FastRandomContext& rng_fast, size_t change_output_size, size_t change_spend_size, + CAmount min_change_target, CFeeRate effective_feerate, + CFeeRate long_term_feerate, CFeeRate discard_feerate, size_t tx_noinputs_size, bool avoid_partial) + : rng_fast{rng_fast}, + change_output_size(change_output_size), + change_spend_size(change_spend_size), + m_min_change_target(min_change_target), + m_effective_feerate(effective_feerate), + m_long_term_feerate(long_term_feerate), + m_discard_feerate(discard_feerate), + tx_noinputs_size(tx_noinputs_size), + m_avoid_partial_spends(avoid_partial) + { + } + CoinSelectionParams(FastRandomContext& rng_fast) + : rng_fast{rng_fast} {} }; /** Parameters for filtering which OutputGroups we may use in coin selection. @@ -139,7 +163,7 @@ struct CoinEligibilityFilter struct OutputGroup { /** The list of UTXOs contained in this output group. */ - std::vector<CInputCoin> m_outputs; + std::vector<COutput> m_outputs; /** Whether the UTXOs were sent by the wallet to itself. This is relevant because we may want at * least a certain number of confirmations on UTXOs received from outside wallets while trusting * our own UTXOs more. */ @@ -176,7 +200,7 @@ struct OutputGroup m_subtract_fee_outputs(params.m_subtract_fee_outputs) {} - void Insert(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants, bool positive_only); + void Insert(const COutput& output, size_t ancestors, size_t descendants, bool positive_only); bool EligibleForSpending(const CoinEligibilityFilter& eligibility_filter) const; CAmount GetSelectionAmount() const; }; @@ -198,23 +222,51 @@ struct OutputGroup * @param[in] use_effective_value Whether to use the input's effective value (when true) or the real value (when false). * @return The waste */ -[[nodiscard]] CAmount GetSelectionWaste(const std::set<CInputCoin>& inputs, CAmount change_cost, CAmount target, bool use_effective_value = true); +[[nodiscard]] CAmount GetSelectionWaste(const std::set<COutput>& inputs, CAmount change_cost, CAmount target, bool use_effective_value = true); + + +/** Choose a random change target for each transaction to make it harder to fingerprint the Core + * wallet based on the change output values of transactions it creates. + * The random value is between 50ksat and min(2 * payment_value, 1milsat) + * When payment_value <= 25ksat, the value is just 50ksat. + * + * Making change amounts similar to the payment value may help disguise which output(s) are payments + * are which ones are change. Using double the payment value may increase the number of inputs + * needed (and thus be more expensive in fees), but breaks analysis techniques which assume the + * coins selected are just sufficient to cover the payment amount ("unnecessary input" heuristic). + * + * @param[in] payment_value Average payment value of the transaction output(s). + */ +[[nodiscard]] CAmount GenerateChangeTarget(CAmount payment_value, FastRandomContext& rng); + +enum class SelectionAlgorithm : uint8_t +{ + BNB = 0, + KNAPSACK = 1, + SRD = 2, + MANUAL = 3, +}; + +std::string GetAlgorithmName(const SelectionAlgorithm algo); struct SelectionResult { private: /** Set of inputs selected by the algorithm to use in the transaction */ - std::set<CInputCoin> m_selected_inputs; - /** The target the algorithm selected for. Note that this may not be equal to the recipient amount as it can include non-input fees */ - const CAmount m_target; + std::set<COutput> m_selected_inputs; /** Whether the input values for calculations should be the effective value (true) or normal value (false) */ bool m_use_effective{false}; /** The computed waste */ std::optional<CAmount> m_waste; public: - explicit SelectionResult(const CAmount target) - : m_target(target) {} + /** The target the algorithm selected for. Note that this may not be equal to the recipient amount as it can include non-input fees */ + const CAmount m_target; + /** The algorithm used to produce this result */ + const SelectionAlgorithm m_algo; + + explicit SelectionResult(const CAmount target, SelectionAlgorithm algo) + : m_target(target), m_algo(algo) {} SelectionResult() = delete; @@ -230,9 +282,9 @@ public: [[nodiscard]] CAmount GetWaste() const; /** Get m_selected_inputs */ - const std::set<CInputCoin>& GetInputSet() const; - /** Get the vector of CInputCoins that will be used to fill in a CTransaction's vin */ - std::vector<CInputCoin> GetShuffledInputVector() const; + const std::set<COutput>& GetInputSet() const; + /** Get the vector of COutputs that will be used to fill in a CTransaction's vin */ + std::vector<COutput> GetShuffledInputVector() const; bool operator<(SelectionResult other) const; }; @@ -246,10 +298,11 @@ std::optional<SelectionResult> SelectCoinsBnB(std::vector<OutputGroup>& utxo_poo * @param[in] target_value The target value to select for * @returns If successful, a SelectionResult, otherwise, std::nullopt */ -std::optional<SelectionResult> SelectCoinsSRD(const std::vector<OutputGroup>& utxo_pool, CAmount target_value); +std::optional<SelectionResult> SelectCoinsSRD(const std::vector<OutputGroup>& utxo_pool, CAmount target_value, FastRandomContext& rng); // Original coin selection algorithm as a fallback -std::optional<SelectionResult> KnapsackSolver(std::vector<OutputGroup>& groups, const CAmount& nTargetValue); +std::optional<SelectionResult> KnapsackSolver(std::vector<OutputGroup>& groups, const CAmount& nTargetValue, + CAmount change_target, FastRandomContext& rng); } // namespace wallet #endif // BITCOIN_WALLET_COINSELECTION_H diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 0ed2658129..8e79ee678e 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -6,6 +6,7 @@ #include <chainparams.h> #include <fs.h> #include <logging.h> +#include <util/system.h> #include <wallet/db.h> #include <exception> @@ -137,4 +138,13 @@ bool IsSQLiteFile(const fs::path& path) // Check the application id matches our network magic return memcmp(Params().MessageStart(), app_id, 4) == 0; } + +void ReadDatabaseArgs(const ArgsManager& args, DatabaseOptions& options) +{ + // Override current options with args values, if any were specified + options.use_unsafe_sync = args.GetBoolArg("-unsafesqlitesync", options.use_unsafe_sync); + options.use_shared_memory = !args.GetBoolArg("-privdb", !options.use_shared_memory); + options.max_log_mb = args.GetIntArg("-dblogsize", options.max_log_mb); +} + } // namespace wallet diff --git a/src/wallet/db.h b/src/wallet/db.h index 5825b00e3a..f09844c37e 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -16,6 +16,7 @@ #include <optional> #include <string> +class ArgsManager; struct bilingual_str; namespace wallet { @@ -207,7 +208,12 @@ struct DatabaseOptions { std::optional<DatabaseFormat> require_format; uint64_t create_flags = 0; SecureString create_passphrase; - bool verify = true; + + // Specialized options. Not every option is supported by every backend. + bool verify = true; //!< Check data integrity on load. + bool use_unsafe_sync = false; //!< Disable file sync for faster performance. + bool use_shared_memory = false; //!< Let other processes access the database. + int64_t max_log_mb = 100; //!< Max log size to allow before consolidating. }; enum class DatabaseStatus { @@ -227,6 +233,7 @@ enum class DatabaseStatus { /** Recursively list database paths in directory. */ std::vector<fs::path> ListDatabases(const fs::path& path); +void ReadDatabaseArgs(const ArgsManager& args, DatabaseOptions& options); std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); fs::path BDBDataFile(const fs::path& path); diff --git a/src/wallet/dump.cpp b/src/wallet/dump.cpp index 6d8508fc72..d80c3e25b0 100644 --- a/src/wallet/dump.cpp +++ b/src/wallet/dump.cpp @@ -19,10 +19,10 @@ namespace wallet { static const std::string DUMP_MAGIC = "BITCOIN_CORE_WALLET_DUMP"; uint32_t DUMP_VERSION = 1; -bool DumpWallet(CWallet& wallet, bilingual_str& error) +bool DumpWallet(const ArgsManager& args, CWallet& wallet, bilingual_str& error) { // Get the dumpfile - std::string dump_filename = gArgs.GetArg("-dumpfile", ""); + std::string dump_filename = args.GetArg("-dumpfile", ""); if (dump_filename.empty()) { error = _("No dump file provided. To use dump, -dumpfile=<filename> must be provided."); return false; @@ -114,10 +114,10 @@ static void WalletToolReleaseWallet(CWallet* wallet) delete wallet; } -bool CreateFromDump(const std::string& name, const fs::path& wallet_path, bilingual_str& error, std::vector<bilingual_str>& warnings) +bool CreateFromDump(const ArgsManager& args, const std::string& name, const fs::path& wallet_path, bilingual_str& error, std::vector<bilingual_str>& warnings) { // Get the dumpfile - std::string dump_filename = gArgs.GetArg("-dumpfile", ""); + std::string dump_filename = args.GetArg("-dumpfile", ""); if (dump_filename.empty()) { error = _("No dump file provided. To use createfromdump, -dumpfile=<filename> must be provided."); return false; @@ -171,7 +171,7 @@ bool CreateFromDump(const std::string& name, const fs::path& wallet_path, biling return false; } // Get the data file format with format_value as the default - std::string file_format = gArgs.GetArg("-format", format_value); + std::string file_format = args.GetArg("-format", format_value); if (file_format.empty()) { error = _("No wallet file format provided. To use createfromdump, -format=<format> must be provided."); return false; @@ -193,6 +193,7 @@ bool CreateFromDump(const std::string& name, const fs::path& wallet_path, biling DatabaseOptions options; DatabaseStatus status; + ReadDatabaseArgs(args, options); options.require_create = true; options.require_format = data_format; std::unique_ptr<WalletDatabase> database = MakeDatabase(wallet_path, options, status, error); diff --git a/src/wallet/dump.h b/src/wallet/dump.h index a879c4db35..bf683e9843 100644 --- a/src/wallet/dump.h +++ b/src/wallet/dump.h @@ -11,11 +11,12 @@ #include <vector> struct bilingual_str; +class ArgsManager; namespace wallet { class CWallet; -bool DumpWallet(CWallet& wallet, bilingual_str& error); -bool CreateFromDump(const std::string& name, const fs::path& wallet_path, bilingual_str& error, std::vector<bilingual_str>& warnings); +bool DumpWallet(const ArgsManager& args, CWallet& wallet, bilingual_str& error); +bool CreateFromDump(const ArgsManager& args, const std::string& name, const fs::path& wallet_path, bilingual_str& error, std::vector<bilingual_str>& warnings); } // namespace wallet #endif // BITCOIN_WALLET_DUMP_H diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 3552c14160..73042424ad 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -135,7 +135,7 @@ static CFeeRate EstimateFeeRate(const CWallet& wallet, const CWalletTx& wtx, con feerate += std::max(node_incremental_relay_fee, wallet_incremental_relay_fee); // Fee rate must also be at least the wallet's GetMinimumFeeRate - CFeeRate min_feerate(GetMinimumFeeRate(wallet, coin_control, /* feeCalc */ nullptr)); + CFeeRate min_feerate(GetMinimumFeeRate(wallet, coin_control, /*feeCalc=*/nullptr)); // Set the required fee rate for the replacement transaction in coin control. return std::max(feerate, min_feerate); @@ -233,12 +233,6 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo // Write back transaction mtx = CMutableTransaction(*tx_new); - // Mark new tx not replaceable, if requested. - if (!coin_control.m_signal_bip125_rbf.value_or(wallet.m_signal_rbf)) { - for (auto& input : mtx.vin) { - if (input.nSequence < 0xfffffffe) input.nSequence = 0xfffffffe; - } - } return Result::OK; } diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 7a83dbc35d..7f038eda84 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -80,9 +80,9 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const argsman.AddArg("-walletrbf", strprintf("Send transactions with full-RBF opt-in enabled (RPC only, default: %u)", DEFAULT_WALLET_RBF), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); #ifdef USE_BDB - argsman.AddArg("-dblogsize=<n>", strprintf("Flush wallet database activity from memory to disk log every <n> megabytes (default: %u)", DEFAULT_WALLET_DBLOGSIZE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); + argsman.AddArg("-dblogsize=<n>", strprintf("Flush wallet database activity from memory to disk log every <n> megabytes (default: %u)", DatabaseOptions().max_log_mb), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); argsman.AddArg("-flushwallet", strprintf("Run a thread to flush wallet periodically (default: %u)", DEFAULT_FLUSHWALLET), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); - argsman.AddArg("-privdb", strprintf("Sets the DB_PRIVATE flag in the wallet db environment (default: %u)", DEFAULT_WALLET_PRIVDB), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); + argsman.AddArg("-privdb", strprintf("Sets the DB_PRIVATE flag in the wallet db environment (default: %u)", !DatabaseOptions().use_shared_memory), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); #else argsman.AddHiddenArgs({"-dblogsize", "-flushwallet", "-privdb"}); #endif @@ -94,6 +94,7 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const #endif argsman.AddArg("-walletrejectlongchains", strprintf("Wallet will not create transactions that violate mempool chain limits (default: %u)", DEFAULT_WALLET_REJECT_LONG_CHAINS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); + argsman.AddArg("-walletcrosschain", strprintf("Allow reusing wallet files across chains (default: %u)", DEFAULT_WALLETCROSSCHAIN), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); argsman.AddHiddenArgs({"-zapwallettxes"}); } diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index 9083c304b2..98e843385c 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -111,6 +111,17 @@ WalletTxOut MakeWalletTxOut(const CWallet& wallet, return result; } +WalletTxOut MakeWalletTxOut(const CWallet& wallet, + const COutput& output) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +{ + WalletTxOut result; + result.txout = output.txout; + result.time = output.time; + result.depth_in_main_chain = output.depth; + result.is_spent = wallet.IsSpent(output.outpoint.hash, output.outpoint.n); + return result; +} + class WalletImpl : public Wallet { public: @@ -419,8 +430,8 @@ public: for (const auto& entry : ListCoins(*m_wallet)) { auto& group = result[entry.first]; for (const auto& coin : entry.second) { - group.emplace_back(COutPoint(coin.tx->GetHash(), coin.i), - MakeWalletTxOut(*m_wallet, *coin.tx, coin.i, coin.nDepth)); + group.emplace_back(coin.outpoint, + MakeWalletTxOut(*m_wallet, coin)); } } return result; @@ -544,6 +555,7 @@ public: std::shared_ptr<CWallet> wallet; DatabaseOptions options; DatabaseStatus status; + ReadDatabaseArgs(*m_context.args, options); options.require_create = true; options.create_flags = wallet_creation_flags; options.create_passphrase = passphrase; @@ -553,6 +565,7 @@ public: { DatabaseOptions options; DatabaseStatus status; + ReadDatabaseArgs(*m_context.args, options); options.require_existing = true; return MakeWallet(m_context, LoadWallet(m_context, name, true /* load_on_start */, options, status, error, warnings)); } diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 633d8c5450..c06513588b 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -57,6 +57,7 @@ bool VerifyWallets(WalletContext& context) if (!args.IsArgSet("wallet")) { DatabaseOptions options; DatabaseStatus status; + ReadDatabaseArgs(args, options); bilingual_str error_string; options.require_existing = true; options.verify = false; @@ -84,6 +85,7 @@ bool VerifyWallets(WalletContext& context) DatabaseOptions options; DatabaseStatus status; + ReadDatabaseArgs(args, options); options.require_existing = true; options.verify = true; bilingual_str error_string; @@ -112,6 +114,7 @@ bool LoadWallets(WalletContext& context) } DatabaseOptions options; DatabaseStatus status; + ReadDatabaseArgs(*context.args, options); options.require_existing = true; options.verify = false; // No need to verify, assuming verified earlier in VerifyWallets() bilingual_str error; @@ -127,6 +130,8 @@ bool LoadWallets(WalletContext& context) chain.initError(error); return false; } + + NotifyWalletLoaded(context, pwallet); AddWallet(context, pwallet); } return true; diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index 1a6f06213c..cddf94aab2 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -325,8 +325,8 @@ Balance GetBalance(const CWallet& wallet, const int min_depth, bool avoid_reuse) const CWalletTx& wtx = entry.second; const bool is_trusted{CachedTxIsTrusted(wallet, wtx, trusted_parents)}; const int tx_depth{wallet.GetTxDepthInMainChain(wtx)}; - const CAmount tx_credit_mine{CachedTxGetAvailableCredit(wallet, wtx, /* fUseCache */ true, ISMINE_SPENDABLE | reuse_filter)}; - const CAmount tx_credit_watchonly{CachedTxGetAvailableCredit(wallet, wtx, /* fUseCache */ true, ISMINE_WATCH_ONLY | reuse_filter)}; + const CAmount tx_credit_mine{CachedTxGetAvailableCredit(wallet, wtx, /*fUseCache=*/true, ISMINE_SPENDABLE | reuse_filter)}; + const CAmount tx_credit_watchonly{CachedTxGetAvailableCredit(wallet, wtx, /*fUseCache=*/true, ISMINE_WATCH_ONLY | reuse_filter)}; if (is_trusted && tx_depth >= min_depth) { ret.m_mine_trusted += tx_credit_mine; ret.m_watchonly_trusted += tx_credit_watchonly; diff --git a/src/wallet/rpc/addresses.cpp b/src/wallet/rpc/addresses.cpp index 51587a64a3..d4f6c9d805 100644 --- a/src/wallet/rpc/addresses.cpp +++ b/src/wallet/rpc/addresses.cpp @@ -239,7 +239,7 @@ RPCHelpMan addmultisigaddress() {RPCResult::Type::STR, "address", "The value of the new multisig address"}, {RPCResult::Type::STR_HEX, "redeemScript", "The string value of the hex-encoded redemption script"}, {RPCResult::Type::STR, "descriptor", "The descriptor for this multisig"}, - {RPCResult::Type::ARR, "warnings", /* optional */ true, "Any warnings resulting from the creation of this multisig", + {RPCResult::Type::ARR, "warnings", /*optional=*/true, "Any warnings resulting from the creation of this multisig", { {RPCResult::Type::STR, "", ""}, }}, @@ -597,7 +597,7 @@ RPCHelpMan getaddressinfo() DescriptorScriptPubKeyMan* desc_spk_man = dynamic_cast<DescriptorScriptPubKeyMan*>(spk_man); if (desc_spk_man) { std::string desc_str; - if (desc_spk_man->GetDescriptorString(desc_str, /* priv */ false)) { + if (desc_spk_man->GetDescriptorString(desc_str, /*priv=*/false)) { ret.pushKV("parent_desc", desc_str); } } diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 228564fae4..b4f01b00de 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -26,8 +26,6 @@ #include <tuple> #include <string> -#include <boost/algorithm/string.hpp> - #include <univalue.h> @@ -546,8 +544,7 @@ RPCHelpMan importwallet() if (line.empty() || line[0] == '#') continue; - std::vector<std::string> vstr; - boost::split(vstr, line, boost::is_any_of(" ")); + std::vector<std::string> vstr = SplitString(line, ' '); if (vstr.size() < 2) continue; CKey key = DecodeSecret(vstr[0]); @@ -915,7 +912,7 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d case TxoutType::WITNESS_V1_TAPROOT: return "unrecognized script"; } // no default case, so the compiler can warn about missing cases - CHECK_NONFATAL(false); + NONFATAL_UNREACHABLE(); } static UniValue ProcessImportLegacy(ImportData& import_data, std::map<CKeyID, CPubKey>& pubkey_map, std::map<CKeyID, CKey>& privkey_map, std::set<CScript>& script_pub_keys, bool& have_solving_data, const UniValue& data, std::vector<CKeyID>& ordered_pubkeys) @@ -1260,7 +1257,7 @@ RPCHelpMan importmulti() { {"desc", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Descriptor to import. If using descriptor, do not also provide address/scriptPubKey, scripts, or pubkeys"}, {"scriptPubKey", RPCArg::Type::STR, RPCArg::Optional::NO, "Type of scriptPubKey (string for script, json for address). Should not be provided if using a descriptor", - /* oneline_description */ "", {"\"<script>\" | { \"address\":\"<address>\" }", "string / json"} + /*oneline_description=*/"", {"\"<script>\" | { \"address\":\"<address>\" }", "string / json"} }, {"timestamp", RPCArg::Type::NUM, RPCArg::Optional::NO, "Creation time of the key expressed in " + UNIX_EPOCH_TIME + ",\n" " or the string \"now\" to substitute the current synced blockchain time. The timestamp of the oldest\n" @@ -1268,7 +1265,7 @@ RPCHelpMan importmulti() " \"now\" can be specified to bypass scanning, for keys which are known to never have been used, and\n" " 0 can be specified to scan the entire blockchain. Blocks up to 2 hours before the earliest key\n" " creation time of all keys being imported by the importmulti call will be scanned.", - /* oneline_description */ "", {"timestamp | \"now\"", "integer / string"} + /*oneline_description=*/"", {"timestamp | \"now\"", "integer / string"} }, {"redeemscript", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Allowed only if the scriptPubKey is a P2SH or P2SH-P2WSH address/scriptPubKey"}, {"witnessscript", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Allowed only if the scriptPubKey is a P2SH-P2WSH or P2WSH address/scriptPubKey"}, @@ -1596,7 +1593,7 @@ RPCHelpMan importdescriptors() " \"now\" can be specified to bypass scanning, for outputs which are known to never have been used, and\n" " 0 can be specified to scan the entire blockchain. Blocks up to 2 hours before the earliest timestamp\n" " of all descriptors being imported will be scanned.", - /* oneline_description */ "", {"timestamp | \"now\"", "integer / string"} + /*oneline_description=*/"", {"timestamp | \"now\"", "integer / string"} }, {"internal", RPCArg::Type::BOOL, RPCArg::Default{false}, "Whether matching outputs should be treated as not incoming payments (e.g. change)"}, {"label", RPCArg::Type::STR, RPCArg::Default{""}, "Label to assign to the address, only allowed with internal=false. Disabled for ranged descriptors"}, @@ -1748,13 +1745,13 @@ RPCHelpMan listdescriptors() {RPCResult::Type::OBJ, "", "", { {RPCResult::Type::STR, "desc", "Descriptor string representation"}, {RPCResult::Type::NUM, "timestamp", "The creation time of the descriptor"}, - {RPCResult::Type::BOOL, "active", "Activeness flag"}, - {RPCResult::Type::BOOL, "internal", true, "Whether this is an internal or external descriptor; defined only for active descriptors"}, - {RPCResult::Type::ARR_FIXED, "range", true, "Defined only for ranged descriptors", { + {RPCResult::Type::BOOL, "active", "Whether this descriptor is currently used to generate new addresses"}, + {RPCResult::Type::BOOL, "internal", /*optional=*/true, "True if this descriptor is used to generate change addresses. False if this descriptor is used to generate receiving addresses; defined only for active descriptors"}, + {RPCResult::Type::ARR_FIXED, "range", /*optional=*/true, "Defined only for ranged descriptors", { {RPCResult::Type::NUM, "", "Range start inclusive"}, {RPCResult::Type::NUM, "", "Range end inclusive"}, }}, - {RPCResult::Type::NUM, "next", true, "The next index to generate addresses from; defined only for ranged descriptors"}, + {RPCResult::Type::NUM, "next", /*optional=*/true, "The next index to generate addresses from; defined only for ranged descriptors"}, }}, }} }}, diff --git a/src/wallet/rpc/coins.cpp b/src/wallet/rpc/coins.cpp index 035541babd..4eccff3969 100644 --- a/src/wallet/rpc/coins.cpp +++ b/src/wallet/rpc/coins.cpp @@ -112,7 +112,7 @@ RPCHelpMan getreceivedbyaddress() LOCK(pwallet->cs_wallet); - return ValueFromAmount(GetReceived(*pwallet, request.params, /* by_label */ false)); + return ValueFromAmount(GetReceived(*pwallet, request.params, /*by_label=*/false)); }, }; } @@ -153,7 +153,7 @@ RPCHelpMan getreceivedbylabel() LOCK(pwallet->cs_wallet); - return ValueFromAmount(GetReceived(*pwallet, request.params, /* by_label */ true)); + return ValueFromAmount(GetReceived(*pwallet, request.params, /*by_label=*/true)); }, }; } @@ -648,16 +648,16 @@ RPCHelpMan listunspent() for (const COutput& out : vecOutputs) { CTxDestination address; - const CScript& scriptPubKey = out.tx->tx->vout[out.i].scriptPubKey; + const CScript& scriptPubKey = out.txout.scriptPubKey; bool fValidAddress = ExtractDestination(scriptPubKey, address); - bool reused = avoid_reuse && pwallet->IsSpentKey(out.tx->GetHash(), out.i); + bool reused = avoid_reuse && pwallet->IsSpentKey(out.outpoint.hash, out.outpoint.n); if (destinations.size() && (!fValidAddress || !destinations.count(address))) continue; UniValue entry(UniValue::VOBJ); - entry.pushKV("txid", out.tx->GetHash().GetHex()); - entry.pushKV("vout", out.i); + entry.pushKV("txid", out.outpoint.hash.GetHex()); + entry.pushKV("vout", (int)out.outpoint.n); if (fValidAddress) { entry.pushKV("address", EncodeDestination(address)); @@ -702,21 +702,21 @@ RPCHelpMan listunspent() } entry.pushKV("scriptPubKey", HexStr(scriptPubKey)); - entry.pushKV("amount", ValueFromAmount(out.tx->tx->vout[out.i].nValue)); - entry.pushKV("confirmations", out.nDepth); - if (!out.nDepth) { + entry.pushKV("amount", ValueFromAmount(out.txout.nValue)); + entry.pushKV("confirmations", out.depth); + if (!out.depth) { size_t ancestor_count, descendant_count, ancestor_size; CAmount ancestor_fees; - pwallet->chain().getTransactionAncestry(out.tx->GetHash(), ancestor_count, descendant_count, &ancestor_size, &ancestor_fees); + pwallet->chain().getTransactionAncestry(out.outpoint.hash, ancestor_count, descendant_count, &ancestor_size, &ancestor_fees); if (ancestor_count) { entry.pushKV("ancestorcount", uint64_t(ancestor_count)); entry.pushKV("ancestorsize", uint64_t(ancestor_size)); entry.pushKV("ancestorfees", uint64_t(ancestor_fees)); } } - entry.pushKV("spendable", out.fSpendable); - entry.pushKV("solvable", out.fSolvable); - if (out.fSolvable) { + entry.pushKV("spendable", out.spendable); + entry.pushKV("solvable", out.solvable); + if (out.solvable) { std::unique_ptr<SigningProvider> provider = pwallet->GetSolvingProvider(scriptPubKey); if (provider) { auto descriptor = InferDescriptor(scriptPubKey, *provider); @@ -724,7 +724,7 @@ RPCHelpMan listunspent() } } if (avoid_reuse) entry.pushKV("reused", reused); - entry.pushKV("safe", out.fSafe); + entry.pushKV("safe", out.safe); results.push_back(entry); } diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index 433b5a1815..07119133b7 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -9,10 +9,12 @@ #include <rpc/rawtransaction_util.h> #include <rpc/util.h> #include <util/fees.h> +#include <util/rbf.h> #include <util/translation.h> #include <util/vector.h> #include <wallet/coincontrol.h> #include <wallet/feebumper.h> +#include <wallet/fees.h> #include <wallet/rpc/util.h> #include <wallet/spend.h> #include <wallet/wallet.h> @@ -21,7 +23,8 @@ namespace wallet { -static void ParseRecipients(const UniValue& address_amounts, const UniValue& subtract_fee_outputs, std::vector<CRecipient> &recipients) { +static void ParseRecipients(const UniValue& address_amounts, const UniValue& subtract_fee_outputs, std::vector<CRecipient>& recipients) +{ std::set<CTxDestination> destinations; int i = 0; for (const std::string& address: address_amounts.getKeys()) { @@ -51,6 +54,93 @@ static void ParseRecipients(const UniValue& address_amounts, const UniValue& sub } } +static void InterpretFeeEstimationInstructions(const UniValue& conf_target, const UniValue& estimate_mode, const UniValue& fee_rate, UniValue& options) +{ + if (options.exists("conf_target") || options.exists("estimate_mode")) { + if (!conf_target.isNull() || !estimate_mode.isNull()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Pass conf_target and estimate_mode either as arguments or in the options object, but not both"); + } + } else { + options.pushKV("conf_target", conf_target); + options.pushKV("estimate_mode", estimate_mode); + } + if (options.exists("fee_rate")) { + if (!fee_rate.isNull()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Pass the fee_rate either as an argument, or in the options object, but not both"); + } + } else { + options.pushKV("fee_rate", fee_rate); + } + if (!options["conf_target"].isNull() && (options["estimate_mode"].isNull() || (options["estimate_mode"].get_str() == "unset"))) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Specify estimate_mode"); + } +} + +static UniValue FinishTransaction(const std::shared_ptr<CWallet> pwallet, const UniValue& options, const CMutableTransaction& rawTx) +{ + // Make a blank psbt + PartiallySignedTransaction psbtx(rawTx); + + // First fill transaction with our data without signing, + // so external signers are not asked sign more than once. + bool complete; + pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, false, true); + const TransactionError err{pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, true, false)}; + if (err != TransactionError::OK) { + throw JSONRPCTransactionError(err); + } + + CMutableTransaction mtx; + complete = FinalizeAndExtractPSBT(psbtx, mtx); + + UniValue result(UniValue::VOBJ); + + const bool psbt_opt_in{options.exists("psbt") && options["psbt"].get_bool()}; + bool add_to_wallet{options.exists("add_to_wallet") ? options["add_to_wallet"].get_bool() : true}; + if (psbt_opt_in || !complete || !add_to_wallet) { + // Serialize the PSBT + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << psbtx; + result.pushKV("psbt", EncodeBase64(ssTx.str())); + } + + if (complete) { + std::string hex{EncodeHexTx(CTransaction(mtx))}; + CTransactionRef tx(MakeTransactionRef(std::move(mtx))); + result.pushKV("txid", tx->GetHash().GetHex()); + if (add_to_wallet && !psbt_opt_in) { + pwallet->CommitTransaction(tx, {}, /*orderForm=*/{}); + } else { + result.pushKV("hex", hex); + } + } + result.pushKV("complete", complete); + + return result; +} + +static void PreventOutdatedOptions(const UniValue& options) +{ + if (options.exists("feeRate")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Use fee_rate (" + CURRENCY_ATOM + "/vB) instead of feeRate"); + } + if (options.exists("changeAddress")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Use change_address instead of changeAddress"); + } + if (options.exists("changePosition")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Use change_position instead of changePosition"); + } + if (options.exists("includeWatching")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Use include_watching instead of includeWatching"); + } + if (options.exists("lockUnspents")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Use lock_unspents instead of lockUnspents"); + } + if (options.exists("subtractFeeFromOutputs")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Use subtract_fee_from_outputs instead of subtractFeeFromOutputs"); + } +} + UniValue SendMoney(CWallet& wallet, const CCoinControl &coin_control, std::vector<CRecipient> &recipients, mapValue_t map_value, bool verbose) { EnsureWalletIsUnlocked(wallet); @@ -108,7 +198,7 @@ static void SetFeeEstimateMode(const CWallet& wallet, CCoinControl& cc, const Un throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both estimate_mode and fee_rate"); } // Fee rates in sat/vB cannot represent more than 3 significant digits. - cc.m_feerate = CFeeRate{AmountFromValue(fee_rate, /* decimals */ 3)}; + cc.m_feerate = CFeeRate{AmountFromValue(fee_rate, /*decimals=*/3)}; if (override_min_fee) cc.fOverrideFeeRate = true; // Default RBF to true for explicit fee_rate, if unset. if (!cc.m_signal_bip125_rbf) cc.m_signal_bip125_rbf = true; @@ -203,7 +293,7 @@ RPCHelpMan sendtoaddress() // We also enable partial spend avoidance if reuse avoidance is set. coin_control.m_avoid_partial_spends |= coin_control.m_avoid_address_reuse; - SetFeeEstimateMode(*pwallet, coin_control, /* conf_target */ request.params[6], /* estimate_mode */ request.params[7], /* fee_rate */ request.params[9], /* override_min_fee */ false); + SetFeeEstimateMode(*pwallet, coin_control, /*conf_target=*/request.params[6], /*estimate_mode=*/request.params[7], /*fee_rate=*/request.params[9], /*override_min_fee=*/false); EnsureWalletIsUnlocked(*pwallet); @@ -306,7 +396,7 @@ RPCHelpMan sendmany() coin_control.m_signal_bip125_rbf = request.params[5].get_bool(); } - SetFeeEstimateMode(*pwallet, coin_control, /* conf_target */ request.params[6], /* estimate_mode */ request.params[7], /* fee_rate */ request.params[8], /* override_min_fee */ false); + SetFeeEstimateMode(*pwallet, coin_control, /*conf_target=*/request.params[6], /*estimate_mode=*/request.params[7], /*fee_rate=*/request.params[8], /*override_min_fee=*/false); std::vector<CRecipient> recipients; ParseRecipients(sendTo, subtractFeeFromAmount, recipients); @@ -360,31 +450,43 @@ RPCHelpMan settxfee() // Only includes key documentation where the key is snake_case in all RPC methods. MixedCase keys can be added later. -static std::vector<RPCArg> FundTxDoc() +static std::vector<RPCArg> FundTxDoc(bool solving_data = true) { - return { + std::vector<RPCArg> args = { {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks"}, {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, std::string() + "The fee estimate mode, must be one of (case insensitive):\n" " \"" + FeeModes("\"\n\"") + "\""}, - {"replaceable", RPCArg::Type::BOOL, RPCArg::DefaultHint{"wallet default"}, "Marks this transaction as BIP125-replaceable.\n" - "Allows this transaction to be replaced by a transaction with higher fees"}, - {"solving_data", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "Keys and scripts needed for producing a final transaction with a dummy signature.\n" - "Used for fee estimation during coin selection.", - { - {"pubkeys", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Public keys involved in this transaction.", - { - {"pubkey", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A public key"}, - }}, - {"scripts", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Scripts involved in this transaction.", - { - {"script", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A script"}, - }}, - {"descriptors", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Descriptors that provide solving data for this transaction.", - { - {"descriptor", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "A descriptor"}, - }}, - }}, + { + "replaceable", RPCArg::Type::BOOL, RPCArg::DefaultHint{"wallet default"}, "Marks this transaction as BIP125-replaceable.\n" + "Allows this transaction to be replaced by a transaction with higher fees" + }, }; + if (solving_data) { + args.push_back({"solving_data", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "Keys and scripts needed for producing a final transaction with a dummy signature.\n" + "Used for fee estimation during coin selection.", + { + { + "pubkeys", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Public keys involved in this transaction.", + { + {"pubkey", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A public key"}, + } + }, + { + "scripts", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Scripts involved in this transaction.", + { + {"script", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A script"}, + } + }, + { + "descriptors", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Descriptors that provide solving data for this transaction.", + { + {"descriptor", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "A descriptor"}, + } + }, + } + }); + } + return args; } void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, int& change_position, const UniValue& options, CCoinControl& coinControl, bool override_min_fee) @@ -659,7 +761,7 @@ RPCHelpMan fundrawtransaction() {"include_unsafe", RPCArg::Type::BOOL, RPCArg::Default{false}, "Include inputs that are not safe to spend (unconfirmed transactions from outside keys and unconfirmed replacement transactions).\n" "Warning: the resulting transaction may become invalid if one of the unsafe inputs disappears.\n" "If that happens, you will need to fund the transaction with different inputs and republish it."}, - {"changeAddress", RPCArg::Type::STR, RPCArg::DefaultHint{"pool address"}, "The bitcoin address to receive the change"}, + {"changeAddress", RPCArg::Type::STR, RPCArg::DefaultHint{"automatic"}, "The bitcoin address to receive the change"}, {"changePosition", RPCArg::Type::NUM, RPCArg::DefaultHint{"random"}, "The index of the change output"}, {"change_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The output type to use. Only valid if changeAddress is not specified. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, {"includeWatching", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Also select inputs which are watch only.\n" @@ -736,7 +838,7 @@ RPCHelpMan fundrawtransaction() CCoinControl coin_control; // Automatically select (additional) coins. Can be overridden by options.add_inputs. coin_control.m_add_inputs = true; - FundTransaction(*pwallet, tx, fee, change_position, request.params[1], coin_control, /* override_min_fee */ true); + FundTransaction(*pwallet, tx, fee, change_position, request.params[1], coin_control, /*override_min_fee=*/true); UniValue result(UniValue::VOBJ); result.pushKV("hex", EncodeHexTx(CTransaction(tx))); @@ -941,7 +1043,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) if (options.exists("replaceable")) { coin_control.m_signal_bip125_rbf = options["replaceable"].get_bool(); } - SetFeeEstimateMode(*pwallet, coin_control, conf_target, options["estimate_mode"], options["fee_rate"], /* override_min_fee */ false); + SetFeeEstimateMode(*pwallet, coin_control, conf_target, options["estimate_mode"], options["fee_rate"], /*override_min_fee=*/false); } // Make sure the results are valid at least up to the most recent block @@ -1056,7 +1158,7 @@ RPCHelpMan send() "Warning: the resulting transaction may become invalid if one of the unsafe inputs disappears.\n" "If that happens, you will need to fund the transaction with different inputs and republish it."}, {"add_to_wallet", RPCArg::Type::BOOL, RPCArg::Default{true}, "When false, returns a serialized transaction which will not be added to the wallet or broadcast"}, - {"change_address", RPCArg::Type::STR_HEX, RPCArg::DefaultHint{"pool address"}, "The bitcoin address to receive the change"}, + {"change_address", RPCArg::Type::STR, RPCArg::DefaultHint{"automatic"}, "The bitcoin address to receive the change"}, {"change_position", RPCArg::Type::NUM, RPCArg::DefaultHint{"random"}, "The index of the change output"}, {"change_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The output type to use. Only valid if change_address is not specified. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, @@ -1126,102 +1228,249 @@ RPCHelpMan send() if (!pwallet) return NullUniValue; UniValue options{request.params[4].isNull() ? UniValue::VOBJ : request.params[4]}; - if (options.exists("conf_target") || options.exists("estimate_mode")) { - if (!request.params[1].isNull() || !request.params[2].isNull()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Pass conf_target and estimate_mode either as arguments or in the options object, but not both"); - } - } else { - options.pushKV("conf_target", request.params[1]); - options.pushKV("estimate_mode", request.params[2]); - } - if (options.exists("fee_rate")) { - if (!request.params[3].isNull()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Pass the fee_rate either as an argument, or in the options object, but not both"); - } - } else { - options.pushKV("fee_rate", request.params[3]); - } - if (!options["conf_target"].isNull() && (options["estimate_mode"].isNull() || (options["estimate_mode"].get_str() == "unset"))) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Specify estimate_mode"); - } - if (options.exists("feeRate")) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Use fee_rate (" + CURRENCY_ATOM + "/vB) instead of feeRate"); - } - if (options.exists("changeAddress")) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Use change_address"); - } - if (options.exists("changePosition")) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Use change_position"); - } - if (options.exists("includeWatching")) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Use include_watching"); - } - if (options.exists("lockUnspents")) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Use lock_unspents"); - } - if (options.exists("subtractFeeFromOutputs")) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Use subtract_fee_from_outputs"); - } + InterpretFeeEstimationInstructions(/*conf_target=*/request.params[1], /*estimate_mode=*/request.params[2], /*fee_rate=*/request.params[3], options); + PreventOutdatedOptions(options); - const bool psbt_opt_in = options.exists("psbt") && options["psbt"].get_bool(); CAmount fee; int change_position; - bool rbf = pwallet->m_signal_rbf; - if (options.exists("replaceable")) { - rbf = options["replaceable"].get_bool(); - } + bool rbf{options.exists("replaceable") ? options["replaceable"].get_bool() : pwallet->m_signal_rbf}; CMutableTransaction rawTx = ConstructTransaction(options["inputs"], request.params[0], options["locktime"], rbf); CCoinControl coin_control; // Automatically select coins, unless at least one is manually selected. Can // be overridden by options.add_inputs. coin_control.m_add_inputs = rawTx.vin.size() == 0; SetOptionsInputWeights(options["inputs"], options); - FundTransaction(*pwallet, rawTx, fee, change_position, options, coin_control, /* override_min_fee */ false); + FundTransaction(*pwallet, rawTx, fee, change_position, options, coin_control, /*override_min_fee=*/false); - bool add_to_wallet = true; - if (options.exists("add_to_wallet")) { - add_to_wallet = options["add_to_wallet"].get_bool(); + return FinishTransaction(pwallet, options, rawTx); + } + }; +} + +RPCHelpMan sendall() +{ + return RPCHelpMan{"sendall", + "EXPERIMENTAL warning: this call may be changed in future releases.\n" + "\nSpend the value of all (or specific) confirmed UTXOs in the wallet to one or more recipients.\n" + "Unconfirmed inbound UTXOs and locked UTXOs will not be spent. Sendall will respect the avoid_reuse wallet flag.\n" + "If your wallet contains many small inputs, either because it received tiny payments or as a result of accumulating change, consider using `send_max` to exclude inputs that are worth less than the fees needed to spend them.\n", + { + {"recipients", RPCArg::Type::ARR, RPCArg::Optional::NO, "The sendall destinations. Each address may only appear once.\n" + "Optionally some recipients can be specified with an amount to perform payments, but at least one address must appear without a specified amount.\n", + { + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "A bitcoin address which receives an equal share of the unspecified amount."}, + {"", RPCArg::Type::OBJ_USER_KEYS, RPCArg::Optional::OMITTED, "", + { + {"address", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "A key-value pair. The key (string) is the bitcoin address, the value (float or string) is the amount in " + CURRENCY_UNIT + ""}, + }, + }, + }, + }, + {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks"}, + {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, std::string() + "The fee estimate mode, must be one of (case insensitive):\n" + " \"" + FeeModes("\"\n\"") + "\""}, + {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, + { + "options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", + Cat<std::vector<RPCArg>>( + { + {"add_to_wallet", RPCArg::Type::BOOL, RPCArg::Default{true}, "When false, returns the serialized transaction without broadcasting or adding it to the wallet"}, + {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, + {"include_watching", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Also select inputs which are watch-only.\n" + "Only solvable inputs can be used. Watch-only destinations are solvable if the public key and/or output script was imported,\n" + "e.g. with 'importpubkey' or 'importmulti' with the 'pubkeys' or 'desc' field."}, + {"inputs", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Use exactly the specified inputs to build the transaction. Specifying inputs is incompatible with send_max. A JSON array of JSON objects", + { + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id"}, + {"vout", RPCArg::Type::NUM, RPCArg::Optional::NO, "The output number"}, + {"sequence", RPCArg::Type::NUM, RPCArg::Optional::NO, "The sequence number"}, + }, + }, + {"locktime", RPCArg::Type::NUM, RPCArg::Default{0}, "Raw locktime. Non-0 value also locktime-activates inputs"}, + {"lock_unspents", RPCArg::Type::BOOL, RPCArg::Default{false}, "Lock selected unspent outputs"}, + {"psbt", RPCArg::Type::BOOL, RPCArg::DefaultHint{"automatic"}, "Always return a PSBT, implies add_to_wallet=false."}, + {"send_max", RPCArg::Type::BOOL, RPCArg::Default{false}, "When true, only use UTXOs that can pay for their own fees to maximize the output amount. When 'false' (default), no UTXO is left behind. send_max is incompatible with providing specific inputs."}, + }, + FundTxDoc() + ), + "options" + }, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::BOOL, "complete", "If the transaction has a complete set of signatures"}, + {RPCResult::Type::STR_HEX, "txid", /*optional=*/true, "The transaction id for the send. Only 1 transaction is created regardless of the number of addresses."}, + {RPCResult::Type::STR_HEX, "hex", /*optional=*/true, "If add_to_wallet is false, the hex-encoded raw transaction with signature(s)"}, + {RPCResult::Type::STR, "psbt", /*optional=*/true, "If more signatures are needed, or if add_to_wallet is false, the base64-encoded (partially) signed transaction"} + } + }, + RPCExamples{"" + "\nSpend all UTXOs from the wallet with a fee rate of 1 " + CURRENCY_ATOM + "/vB using named arguments\n" + + HelpExampleCli("-named sendall", "recipients='[\"" + EXAMPLE_ADDRESS[0] + "\"]' fee_rate=1\n") + + "Spend all UTXOs with a fee rate of 1.1 " + CURRENCY_ATOM + "/vB using positional arguments\n" + + HelpExampleCli("sendall", "'[\"" + EXAMPLE_ADDRESS[0] + "\"]' null \"unset\" 1.1\n") + + "Spend all UTXOs split into equal amounts to two addresses with a fee rate of 1.5 " + CURRENCY_ATOM + "/vB using the options argument\n" + + HelpExampleCli("sendall", "'[\"" + EXAMPLE_ADDRESS[0] + "\", \"" + EXAMPLE_ADDRESS[1] + "\"]' null \"unset\" null '{\"fee_rate\": 1.5}'\n") + + "Leave dust UTXOs in wallet, spend only UTXOs with positive effective value with a fee rate of 10 " + CURRENCY_ATOM + "/vB using the options argument\n" + + HelpExampleCli("sendall", "'[\"" + EXAMPLE_ADDRESS[0] + "\"]' null \"unset\" null '{\"fee_rate\": 10, \"send_max\": true}'\n") + + "Spend all UTXOs with a fee rate of 1.3 " + CURRENCY_ATOM + "/vB using named arguments and sending a 0.25 " + CURRENCY_UNIT + " to another recipient\n" + + HelpExampleCli("-named sendall", "recipients='[{\"" + EXAMPLE_ADDRESS[1] + "\": 0.25}, \""+ EXAMPLE_ADDRESS[0] + "\"]' fee_rate=1.3\n") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + RPCTypeCheck(request.params, { + UniValue::VARR, // recipients + UniValue::VNUM, // conf_target + UniValue::VSTR, // estimate_mode + UniValueType(), // fee_rate, will be checked by AmountFromValue() in SetFeeEstimateMode() + UniValue::VOBJ, // options + }, true + ); + + std::shared_ptr<CWallet> const pwallet{GetWalletForJSONRPCRequest(request)}; + if (!pwallet) return NullUniValue; + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + + UniValue options{request.params[4].isNull() ? UniValue::VOBJ : request.params[4]}; + InterpretFeeEstimationInstructions(/*conf_target=*/request.params[1], /*estimate_mode=*/request.params[2], /*fee_rate=*/request.params[3], options); + PreventOutdatedOptions(options); + + + std::set<std::string> addresses_without_amount; + UniValue recipient_key_value_pairs(UniValue::VARR); + const UniValue& recipients{request.params[0]}; + for (unsigned int i = 0; i < recipients.size(); ++i) { + const UniValue& recipient{recipients[i]}; + if (recipient.isStr()) { + UniValue rkvp(UniValue::VOBJ); + rkvp.pushKV(recipient.get_str(), 0); + recipient_key_value_pairs.push_back(rkvp); + addresses_without_amount.insert(recipient.get_str()); + } else { + recipient_key_value_pairs.push_back(recipient); + } + } + + if (addresses_without_amount.size() == 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Must provide at least one address without a specified amount"); } - // Make a blank psbt - PartiallySignedTransaction psbtx(rawTx); + CCoinControl coin_control; + + SetFeeEstimateMode(*pwallet, coin_control, options["conf_target"], options["estimate_mode"], options["fee_rate"], /*override_min_fee=*/false); - // First fill transaction with our data without signing, - // so external signers are not asked sign more than once. - bool complete; - pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, false, true); - const TransactionError err = pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, true, false); - if (err != TransactionError::OK) { - throw JSONRPCTransactionError(err); + coin_control.fAllowWatchOnly = ParseIncludeWatchonly(options["include_watching"], *pwallet); + + const bool rbf{options.exists("replaceable") ? options["replaceable"].get_bool() : pwallet->m_signal_rbf}; + + FeeCalculation fee_calc_out; + CFeeRate fee_rate{GetMinimumFeeRate(*pwallet, coin_control, &fee_calc_out)}; + // Do not, ever, assume that it's fine to change the fee rate if the user has explicitly + // provided one + if (coin_control.m_feerate && fee_rate > *coin_control.m_feerate) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Fee rate (%s) is lower than the minimum fee rate setting (%s)", coin_control.m_feerate->ToString(FeeEstimateMode::SAT_VB), fee_rate.ToString(FeeEstimateMode::SAT_VB))); + } + if (fee_calc_out.reason == FeeReason::FALLBACK && !pwallet->m_allow_fallback_fee) { + // eventually allow a fallback fee + throw JSONRPCError(RPC_WALLET_ERROR, "Fee estimation failed. Fallbackfee is disabled. Wait a few blocks or enable -fallbackfee."); } - CMutableTransaction mtx; - complete = FinalizeAndExtractPSBT(psbtx, mtx); + CMutableTransaction rawTx{ConstructTransaction(options["inputs"], recipient_key_value_pairs, options["locktime"], rbf)}; + LOCK(pwallet->cs_wallet); + std::vector<COutput> all_the_utxos; + + CAmount total_input_value(0); + bool send_max{options.exists("send_max") ? options["send_max"].get_bool() : false}; + if (options.exists("inputs") && options.exists("send_max")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot combine send_max with specific inputs."); + } else if (options.exists("inputs")) { + for (const CTxIn& input : rawTx.vin) { + if (pwallet->IsSpent(input.prevout.hash, input.prevout.n)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Input not available. UTXO (%s:%d) was already spent.", input.prevout.hash.ToString(), input.prevout.n)); + } + const CWalletTx* tx{pwallet->GetWalletTx(input.prevout.hash)}; + if (!tx || pwallet->IsMine(tx->tx->vout[input.prevout.n]) != (coin_control.fAllowWatchOnly ? ISMINE_ALL : ISMINE_SPENDABLE)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Input not found. UTXO (%s:%d) is not part of wallet.", input.prevout.hash.ToString(), input.prevout.n)); + } + total_input_value += tx->tx->vout[input.prevout.n].nValue; + } + } else { + AvailableCoins(*pwallet, all_the_utxos, &coin_control, /*nMinimumAmount=*/0); + for (const COutput& output : all_the_utxos) { + CHECK_NONFATAL(output.input_bytes > 0); + if (send_max && fee_rate.GetFee(output.input_bytes) > output.txout.nValue) { + continue; + } + CTxIn input(output.outpoint.hash, output.outpoint.n, CScript(), rbf ? MAX_BIP125_RBF_SEQUENCE : CTxIn::SEQUENCE_FINAL); + rawTx.vin.push_back(input); + total_input_value += output.txout.nValue; + } + } - UniValue result(UniValue::VOBJ); + // estimate final size of tx + const TxSize tx_size{CalculateMaximumSignedTxSize(CTransaction(rawTx), pwallet.get())}; + const CAmount fee_from_size{fee_rate.GetFee(tx_size.vsize)}; + const CAmount effective_value{total_input_value - fee_from_size}; - if (psbt_opt_in || !complete || !add_to_wallet) { - // Serialize the PSBT - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); - ssTx << psbtx; - result.pushKV("psbt", EncodeBase64(ssTx.str())); + if (effective_value <= 0) { + if (send_max) { + throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Total value of UTXO pool too low to pay for transaction, try using lower feerate."); + } else { + throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Total value of UTXO pool too low to pay for transaction. Try using lower feerate or excluding uneconomic UTXOs with 'send_max' option."); + } } - if (complete) { - std::string err_string; - std::string hex = EncodeHexTx(CTransaction(mtx)); - CTransactionRef tx(MakeTransactionRef(std::move(mtx))); - result.pushKV("txid", tx->GetHash().GetHex()); - if (add_to_wallet && !psbt_opt_in) { - pwallet->CommitTransaction(tx, {}, {} /* orderForm */); + CAmount output_amounts_claimed{0}; + for (CTxOut out : rawTx.vout) { + output_amounts_claimed += out.nValue; + } + + if (output_amounts_claimed > total_input_value) { + throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Assigned more value to outputs than available funds."); + } + + const CAmount remainder{effective_value - output_amounts_claimed}; + if (remainder < 0) { + throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Insufficient funds for fees after creating specified outputs."); + } + + const CAmount per_output_without_amount{remainder / (long)addresses_without_amount.size()}; + + bool gave_remaining_to_first{false}; + for (CTxOut& out : rawTx.vout) { + CTxDestination dest; + ExtractDestination(out.scriptPubKey, dest); + std::string addr{EncodeDestination(dest)}; + if (addresses_without_amount.count(addr) > 0) { + out.nValue = per_output_without_amount; + if (!gave_remaining_to_first) { + out.nValue += remainder % addresses_without_amount.size(); + gave_remaining_to_first = true; + } + if (IsDust(out, pwallet->chain().relayDustFee())) { + // Dynamically generated output amount is dust + throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Dynamically assigned remainder results in dust output."); + } } else { - result.pushKV("hex", hex); + if (IsDust(out, pwallet->chain().relayDustFee())) { + // Specified output amount is dust + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Specified output amount to %s is below dust threshold.", addr)); + } + } + } + + const bool lock_unspents{options.exists("lock_unspents") ? options["lock_unspents"].get_bool() : false}; + if (lock_unspents) { + for (const CTxIn& txin : rawTx.vin) { + pwallet->LockCoin(txin.prevout); } } - result.pushKV("complete", complete); - return result; + return FinishTransaction(pwallet, options, rawTx); } }; } @@ -1351,7 +1600,7 @@ RPCHelpMan walletcreatefundedpsbt() {"include_unsafe", RPCArg::Type::BOOL, RPCArg::Default{false}, "Include inputs that are not safe to spend (unconfirmed transactions from outside keys and unconfirmed replacement transactions).\n" "Warning: the resulting transaction may become invalid if one of the unsafe inputs disappears.\n" "If that happens, you will need to fund the transaction with different inputs and republish it."}, - {"changeAddress", RPCArg::Type::STR_HEX, RPCArg::DefaultHint{"pool address"}, "The bitcoin address to receive the change"}, + {"changeAddress", RPCArg::Type::STR, RPCArg::DefaultHint{"automatic"}, "The bitcoin address to receive the change"}, {"changePosition", RPCArg::Type::NUM, RPCArg::DefaultHint{"random"}, "The index of the change output"}, {"change_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The output type to use. Only valid if changeAddress is not specified. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, {"includeWatching", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Also select inputs which are watch only"}, @@ -1418,7 +1667,7 @@ RPCHelpMan walletcreatefundedpsbt() // be overridden by options.add_inputs. coin_control.m_add_inputs = rawTx.vin.size() == 0; SetOptionsInputWeights(request.params[0], options); - FundTransaction(wallet, rawTx, fee, change_position, options, coin_control, /* override_min_fee */ true); + FundTransaction(wallet, rawTx, fee, change_position, options, coin_control, /*override_min_fee=*/true); // Make a blank psbt PartiallySignedTransaction psbtx(rawTx); diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index eef2c13ee1..c87af2ea30 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -34,6 +34,7 @@ static void WalletTxToJSON(const CWallet& wallet, const CWalletTx& wtx, UniValue } uint256 hash = wtx.GetHash(); entry.pushKV("txid", hash.GetHex()); + entry.pushKV("wtxid", wtx.GetWitnessHash().GetHex()); UniValue conflicts(UniValue::VARR); for (const uint256& conflict : wallet.GetTxConflicts(wtx)) conflicts.push_back(conflict.GetHex()); @@ -431,6 +432,7 @@ static const std::vector<RPCResult> TransactionDescriptionString() {RPCResult::Type::NUM, "blockindex", /*optional=*/true, "The index of the transaction in the block that includes it."}, {RPCResult::Type::NUM_TIME, "blocktime", /*optional=*/true, "The block time expressed in " + UNIX_EPOCH_TIME + "."}, {RPCResult::Type::STR_HEX, "txid", "The transaction id."}, + {RPCResult::Type::STR_HEX, "wtxid", "The hash of serialized transaction, including witness data."}, {RPCResult::Type::ARR, "walletconflicts", "Conflicting transaction ids.", { {RPCResult::Type::STR_HEX, "txid", "The transaction id."}, @@ -798,7 +800,7 @@ RPCHelpMan gettransaction() if (verbose) { UniValue decoded(UniValue::VOBJ); - TxToUniv(*wtx.tx, uint256(), decoded, false); + TxToUniv(*wtx.tx, /*block_hash=*/uint256(), /*entry=*/decoded, /*include_hex=*/false); entry.pushKV("decoded", decoded); } diff --git a/src/wallet/rpc/wallet.cpp b/src/wallet/rpc/wallet.cpp index 883a3c102b..efb4d8fc7e 100644 --- a/src/wallet/rpc/wallet.cpp +++ b/src/wallet/rpc/wallet.cpp @@ -8,6 +8,7 @@ #include <rpc/server.h> #include <rpc/util.h> #include <util/translation.h> +#include <wallet/context.h> #include <wallet/receive.h> #include <wallet/rpc/wallet.h> #include <wallet/rpc/util.h> @@ -55,7 +56,7 @@ static RPCHelpMan getwalletinfo() { {RPCResult::Type::NUM, "duration", "elapsed seconds since scan start"}, {RPCResult::Type::NUM, "progress", "scanning progress percentage [0.0, 1.0]"}, - }}, + }, /*skip_type_check=*/true}, {RPCResult::Type::BOOL, "descriptors", "whether this wallet uses descriptors for scriptPubKey management"}, {RPCResult::Type::BOOL, "external_signer", "whether this wallet is configured to use an external signer such as a hardware wallet"}, }}, @@ -220,6 +221,7 @@ static RPCHelpMan loadwallet() DatabaseOptions options; DatabaseStatus status; + ReadDatabaseArgs(*context.args, options); options.require_existing = true; bilingual_str error; std::vector<bilingual_str> warnings; @@ -255,7 +257,7 @@ static RPCHelpMan setwalletflag() { {RPCResult::Type::STR, "flag_name", "The name of the flag that was modified"}, {RPCResult::Type::BOOL, "flag_state", "The new state of the flag"}, - {RPCResult::Type::STR, "warnings", "Any warnings associated with the change"}, + {RPCResult::Type::STR, "warnings", /*optional=*/true, "Any warnings associated with the change"}, } }, RPCExamples{ @@ -315,7 +317,9 @@ static RPCHelpMan createwallet() {"blank", RPCArg::Type::BOOL, RPCArg::Default{false}, "Create a blank wallet. A blank wallet has no keys or HD seed. One can be set using sethdseed."}, {"passphrase", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "Encrypt the wallet with this passphrase."}, {"avoid_reuse", RPCArg::Type::BOOL, RPCArg::Default{false}, "Keep track of coin reuse, and treat dirty and clean coins differently with privacy considerations in mind."}, - {"descriptors", RPCArg::Type::BOOL, RPCArg::Default{true}, "Create a native descriptor wallet. The wallet will use descriptors internally to handle address creation"}, + {"descriptors", RPCArg::Type::BOOL, RPCArg::Default{true}, "Create a native descriptor wallet. The wallet will use descriptors internally to handle address creation." + " Setting to \"false\" will create a legacy wallet; however, the legacy wallet type is being deprecated and" + " support for creating and opening legacy wallets will be removed in the future."}, {"load_on_startup", RPCArg::Type::BOOL, RPCArg::Optional::OMITTED_NAMED_ARG, "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, {"external_signer", RPCArg::Type::BOOL, RPCArg::Default{false}, "Use an external signer such as a hardware wallet. Requires -signer to be configured. Wallet creation will fail if keys cannot be fetched. Requires disable_private_keys and descriptors set to true."}, }, @@ -379,6 +383,7 @@ static RPCHelpMan createwallet() DatabaseOptions options; DatabaseStatus status; + ReadDatabaseArgs(*context.args, options); options.require_create = true; options.create_flags = flags; options.create_passphrase = passphrase; @@ -639,6 +644,7 @@ RPCHelpMan fundrawtransaction(); RPCHelpMan bumpfee(); RPCHelpMan psbtbumpfee(); RPCHelpMan send(); +RPCHelpMan sendall(); RPCHelpMan walletprocesspsbt(); RPCHelpMan walletcreatefundedpsbt(); RPCHelpMan signrawtransactionwithwallet(); @@ -658,78 +664,75 @@ RPCHelpMan abortrescan(); Span<const CRPCCommand> GetWalletRPCCommands() { -// clang-format off -static const CRPCCommand commands[] = -{ // category actor (function) - // ------------------ ------------------------ - { "rawtransactions", &fundrawtransaction, }, - { "wallet", &abandontransaction, }, - { "wallet", &abortrescan, }, - { "wallet", &addmultisigaddress, }, - { "wallet", &backupwallet, }, - { "wallet", &bumpfee, }, - { "wallet", &psbtbumpfee, }, - { "wallet", &createwallet, }, - { "wallet", &restorewallet, }, - { "wallet", &dumpprivkey, }, - { "wallet", &dumpwallet, }, - { "wallet", &encryptwallet, }, - { "wallet", &getaddressesbylabel, }, - { "wallet", &getaddressinfo, }, - { "wallet", &getbalance, }, - { "wallet", &getnewaddress, }, - { "wallet", &getrawchangeaddress, }, - { "wallet", &getreceivedbyaddress, }, - { "wallet", &getreceivedbylabel, }, - { "wallet", &gettransaction, }, - { "wallet", &getunconfirmedbalance, }, - { "wallet", &getbalances, }, - { "wallet", &getwalletinfo, }, - { "wallet", &importaddress, }, - { "wallet", &importdescriptors, }, - { "wallet", &importmulti, }, - { "wallet", &importprivkey, }, - { "wallet", &importprunedfunds, }, - { "wallet", &importpubkey, }, - { "wallet", &importwallet, }, - { "wallet", &keypoolrefill, }, - { "wallet", &listaddressgroupings, }, - { "wallet", &listdescriptors, }, - { "wallet", &listlabels, }, - { "wallet", &listlockunspent, }, - { "wallet", &listreceivedbyaddress, }, - { "wallet", &listreceivedbylabel, }, - { "wallet", &listsinceblock, }, - { "wallet", &listtransactions, }, - { "wallet", &listunspent, }, - { "wallet", &listwalletdir, }, - { "wallet", &listwallets, }, - { "wallet", &loadwallet, }, - { "wallet", &lockunspent, }, - { "wallet", &newkeypool, }, - { "wallet", &removeprunedfunds, }, - { "wallet", &rescanblockchain, }, - { "wallet", &send, }, - { "wallet", &sendmany, }, - { "wallet", &sendtoaddress, }, - { "wallet", &sethdseed, }, - { "wallet", &setlabel, }, - { "wallet", &settxfee, }, - { "wallet", &setwalletflag, }, - { "wallet", &signmessage, }, - { "wallet", &signrawtransactionwithwallet, }, - { "wallet", &unloadwallet, }, - { "wallet", &upgradewallet, }, - { "wallet", &walletcreatefundedpsbt, }, + static const CRPCCommand commands[]{ + {"rawtransactions", &fundrawtransaction}, + {"wallet", &abandontransaction}, + {"wallet", &abortrescan}, + {"wallet", &addmultisigaddress}, + {"wallet", &backupwallet}, + {"wallet", &bumpfee}, + {"wallet", &psbtbumpfee}, + {"wallet", &createwallet}, + {"wallet", &restorewallet}, + {"wallet", &dumpprivkey}, + {"wallet", &dumpwallet}, + {"wallet", &encryptwallet}, + {"wallet", &getaddressesbylabel}, + {"wallet", &getaddressinfo}, + {"wallet", &getbalance}, + {"wallet", &getnewaddress}, + {"wallet", &getrawchangeaddress}, + {"wallet", &getreceivedbyaddress}, + {"wallet", &getreceivedbylabel}, + {"wallet", &gettransaction}, + {"wallet", &getunconfirmedbalance}, + {"wallet", &getbalances}, + {"wallet", &getwalletinfo}, + {"wallet", &importaddress}, + {"wallet", &importdescriptors}, + {"wallet", &importmulti}, + {"wallet", &importprivkey}, + {"wallet", &importprunedfunds}, + {"wallet", &importpubkey}, + {"wallet", &importwallet}, + {"wallet", &keypoolrefill}, + {"wallet", &listaddressgroupings}, + {"wallet", &listdescriptors}, + {"wallet", &listlabels}, + {"wallet", &listlockunspent}, + {"wallet", &listreceivedbyaddress}, + {"wallet", &listreceivedbylabel}, + {"wallet", &listsinceblock}, + {"wallet", &listtransactions}, + {"wallet", &listunspent}, + {"wallet", &listwalletdir}, + {"wallet", &listwallets}, + {"wallet", &loadwallet}, + {"wallet", &lockunspent}, + {"wallet", &newkeypool}, + {"wallet", &removeprunedfunds}, + {"wallet", &rescanblockchain}, + {"wallet", &send}, + {"wallet", &sendmany}, + {"wallet", &sendtoaddress}, + {"wallet", &sethdseed}, + {"wallet", &setlabel}, + {"wallet", &settxfee}, + {"wallet", &setwalletflag}, + {"wallet", &signmessage}, + {"wallet", &signrawtransactionwithwallet}, + {"wallet", &sendall}, + {"wallet", &unloadwallet}, + {"wallet", &upgradewallet}, + {"wallet", &walletcreatefundedpsbt}, #ifdef ENABLE_EXTERNAL_SIGNER - { "wallet", &walletdisplayaddress, }, + {"wallet", &walletdisplayaddress}, #endif // ENABLE_EXTERNAL_SIGNER - { "wallet", &walletlock, }, - { "wallet", &walletpassphrase, }, - { "wallet", &walletpassphrasechange, }, - { "wallet", &walletprocesspsbt, }, -}; -// clang-format on + {"wallet", &walletlock}, + {"wallet", &walletpassphrase}, + {"wallet", &walletpassphrasechange}, + {"wallet", &walletprocesspsbt}, + }; return commands; } } // namespace wallet diff --git a/src/wallet/salvage.cpp b/src/wallet/salvage.cpp index 1ecc96fe0e..9ba3c7fd2c 100644 --- a/src/wallet/salvage.cpp +++ b/src/wallet/salvage.cpp @@ -23,10 +23,11 @@ static bool KeyFilter(const std::string& type) return WalletBatch::IsKeyType(type) || type == DBKeys::HDCHAIN; } -bool RecoverDatabaseFile(const fs::path& file_path, bilingual_str& error, std::vector<bilingual_str>& warnings) +bool RecoverDatabaseFile(const ArgsManager& args, const fs::path& file_path, bilingual_str& error, std::vector<bilingual_str>& warnings) { DatabaseOptions options; DatabaseStatus status; + ReadDatabaseArgs(args, options); options.require_existing = true; options.verify = false; options.require_format = DatabaseFormat::BERKELEY; diff --git a/src/wallet/salvage.h b/src/wallet/salvage.h index 332aceb262..e4822c3c75 100644 --- a/src/wallet/salvage.h +++ b/src/wallet/salvage.h @@ -9,10 +9,11 @@ #include <fs.h> #include <streams.h> +class ArgsManager; struct bilingual_str; namespace wallet { -bool RecoverDatabaseFile(const fs::path& file_path, bilingual_str& error, std::vector<bilingual_str>& warnings); +bool RecoverDatabaseFile(const ArgsManager& args, const fs::path& file_path, bilingual_str& error, std::vector<bilingual_str>& warnings); } // namespace wallet #endif // BITCOIN_WALLET_SALVAGE_H diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 83eaececc1..55c0a2cb7f 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -11,6 +11,7 @@ #include <util/fees.h> #include <util/moneystr.h> #include <util/rbf.h> +#include <util/trace.h> #include <util/translation.h> #include <wallet/coincontrol.h> #include <wallet/fees.h> @@ -19,6 +20,8 @@ #include <wallet/transaction.h> #include <wallet/wallet.h> +#include <cmath> + using interfaces::FoundBlock; namespace wallet { @@ -29,11 +32,6 @@ int GetTxSpendSize(const CWallet& wallet, const CWalletTx& wtx, unsigned int out return CalculateMaximumSignedInputSize(wtx.tx->vout[out], &wallet, use_max_sig); } -std::string COutput::ToString() const -{ - return strprintf("COutput(%s, %d, %d) [%s]", tx->GetHash().ToString(), i, nDepth, FormatMoney(tx->tx->vout[i].nValue)); -} - int CalculateMaximumSignedInputSize(const CTxOut& txout, const SigningProvider* provider, bool use_max_sig) { CMutableTransaction txn; @@ -158,6 +156,8 @@ void AvailableCoins(const CWallet& wallet, std::vector<COutput>& vCoins, const C continue; } + bool tx_from_me = CachedTxIsFromMe(wallet, wtx, ISMINE_ALL); + for (unsigned int i = 0; i < wtx.tx->vout.size(); i++) { // Only consider selected coins if add_inputs is false if (coinControl && !coinControl->m_add_inputs && !coinControl->IsSelected(COutPoint(entry.first, i))) { @@ -190,8 +190,9 @@ void AvailableCoins(const CWallet& wallet, std::vector<COutput>& vCoins, const C bool solvable = provider ? IsSolvable(*provider, wtx.tx->vout[i].scriptPubKey) : false; bool spendable = ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || (((mine & ISMINE_WATCH_ONLY) != ISMINE_NO) && (coinControl && coinControl->fAllowWatchOnly && solvable)); + int input_bytes = GetTxSpendSize(wallet, wtx, i, (coinControl && coinControl->fAllowWatchOnly)); - vCoins.push_back(COutput(wallet, wtx, i, nDepth, spendable, solvable, safeTx, (coinControl && coinControl->fAllowWatchOnly))); + vCoins.emplace_back(COutPoint(wtx.GetHash(), i), wtx.tx->vout.at(i), nDepth, input_bytes, spendable, solvable, safeTx, wtx.GetTxTime(), tx_from_me); // Checks the sum amount of all UTXO's. if (nMinimumSumAmount != MAX_MONEY) { @@ -218,8 +219,8 @@ CAmount GetAvailableBalance(const CWallet& wallet, const CCoinControl* coinContr std::vector<COutput> vCoins; AvailableCoins(wallet, vCoins, coinControl); for (const COutput& out : vCoins) { - if (out.fSpendable) { - balance += out.tx->tx->vout[out.i].nValue; + if (out.spendable) { + balance += out.txout.nValue; } } return balance; @@ -243,6 +244,12 @@ const CTxOut& FindNonChangeParentOutput(const CWallet& wallet, const CTransactio return ptx->vout[n]; } +const CTxOut& FindNonChangeParentOutput(const CWallet& wallet, const COutPoint& outpoint) +{ + AssertLockHeld(wallet.cs_wallet); + return FindNonChangeParentOutput(wallet, *wallet.GetWalletTx(outpoint.hash)->tx, outpoint.n); +} + std::map<CTxDestination, std::vector<COutput>> ListCoins(const CWallet& wallet) { AssertLockHeld(wallet.cs_wallet); @@ -254,8 +261,8 @@ std::map<CTxDestination, std::vector<COutput>> ListCoins(const CWallet& wallet) for (const COutput& coin : availableCoins) { CTxDestination address; - if ((coin.fSpendable || (wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && coin.fSolvable)) && - ExtractDestination(FindNonChangeParentOutput(wallet, *coin.tx->tx, coin.i).scriptPubKey, address)) { + if ((coin.spendable || (wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && coin.solvable)) && + ExtractDestination(FindNonChangeParentOutput(wallet, coin.outpoint).scriptPubKey, address)) { result[address].emplace_back(std::move(coin)); } } @@ -268,14 +275,15 @@ std::map<CTxDestination, std::vector<COutput>> ListCoins(const CWallet& wallet) for (const COutPoint& output : lockedCoins) { auto it = wallet.mapWallet.find(output.hash); if (it != wallet.mapWallet.end()) { - int depth = wallet.GetTxDepthInMainChain(it->second); - if (depth >= 0 && output.n < it->second.tx->vout.size() && - wallet.IsMine(it->second.tx->vout[output.n]) == is_mine_filter + const auto& wtx = it->second; + int depth = wallet.GetTxDepthInMainChain(wtx); + if (depth >= 0 && output.n < wtx.tx->vout.size() && + wallet.IsMine(wtx.tx->vout[output.n]) == is_mine_filter ) { CTxDestination address; - if (ExtractDestination(FindNonChangeParentOutput(wallet, *it->second.tx, output.n).scriptPubKey, address)) { + if (ExtractDestination(FindNonChangeParentOutput(wallet, *wtx.tx, output.n).scriptPubKey, address)) { result[address].emplace_back( - wallet, it->second, output.n, depth, true /* spendable */, true /* solvable */, false /* safe */); + COutPoint(wtx.GetHash(), output.n), wtx.tx->vout.at(output.n), depth, GetTxSpendSize(wallet, wtx, output.n), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ false, wtx.GetTxTime(), CachedTxIsFromMe(wallet, wtx, ISMINE_ALL)); } } } @@ -292,15 +300,14 @@ std::vector<OutputGroup> GroupOutputs(const CWallet& wallet, const std::vector<C // Allowing partial spends means no grouping. Each COutput gets its own OutputGroup. for (const COutput& output : outputs) { // Skip outputs we cannot spend - if (!output.fSpendable) continue; + if (!output.spendable) continue; size_t ancestors, descendants; - wallet.chain().getTransactionAncestry(output.tx->GetHash(), ancestors, descendants); - CInputCoin input_coin = output.GetInputCoin(); + wallet.chain().getTransactionAncestry(output.outpoint.hash, ancestors, descendants); // Make an OutputGroup containing just this output OutputGroup group{coin_sel_params}; - group.Insert(input_coin, output.nDepth, CachedTxIsFromMe(wallet, *output.tx, ISMINE_ALL), ancestors, descendants, positive_only); + group.Insert(output, ancestors, descendants, positive_only); // Check the OutputGroup's eligibility. Only add the eligible ones. if (positive_only && group.GetSelectionAmount() <= 0) continue; @@ -312,18 +319,17 @@ std::vector<OutputGroup> GroupOutputs(const CWallet& wallet, const std::vector<C // We want to combine COutputs that have the same scriptPubKey into single OutputGroups // except when there are more than OUTPUT_GROUP_MAX_ENTRIES COutputs grouped in an OutputGroup. // To do this, we maintain a map where the key is the scriptPubKey and the value is a vector of OutputGroups. - // For each COutput, we check if the scriptPubKey is in the map, and if it is, the COutput's CInputCoin is added + // For each COutput, we check if the scriptPubKey is in the map, and if it is, the COutput is added // to the last OutputGroup in the vector for the scriptPubKey. When the last OutputGroup has - // OUTPUT_GROUP_MAX_ENTRIES CInputCoins, a new OutputGroup is added to the end of the vector. + // OUTPUT_GROUP_MAX_ENTRIES COutputs, a new OutputGroup is added to the end of the vector. std::map<CScript, std::vector<OutputGroup>> spk_to_groups_map; for (const auto& output : outputs) { // Skip outputs we cannot spend - if (!output.fSpendable) continue; + if (!output.spendable) continue; size_t ancestors, descendants; - wallet.chain().getTransactionAncestry(output.tx->GetHash(), ancestors, descendants); - CInputCoin input_coin = output.GetInputCoin(); - CScript spk = input_coin.txout.scriptPubKey; + wallet.chain().getTransactionAncestry(output.outpoint.hash, ancestors, descendants); + CScript spk = output.txout.scriptPubKey; std::vector<OutputGroup>& groups = spk_to_groups_map[spk]; @@ -332,7 +338,7 @@ std::vector<OutputGroup> GroupOutputs(const CWallet& wallet, const std::vector<C groups.emplace_back(coin_sel_params); } - // Get the last OutputGroup in the vector so that we can add the CInputCoin to it + // Get the last OutputGroup in the vector so that we can add the COutput to it // A pointer is used here so that group can be reassigned later if it is full. OutputGroup* group = &groups.back(); @@ -344,8 +350,8 @@ std::vector<OutputGroup> GroupOutputs(const CWallet& wallet, const std::vector<C group = &groups.back(); } - // Add the input_coin to group - group->Insert(input_coin, output.nDepth, CachedTxIsFromMe(wallet, *output.tx, ISMINE_ALL), ancestors, descendants, positive_only); + // Add the output to group + group->Insert(output, ancestors, descendants, positive_only); } // Now we go through the entire map and pull out the OutputGroups @@ -379,7 +385,6 @@ std::optional<SelectionResult> AttemptSelection(const CWallet& wallet, const CAm // Note that unlike KnapsackSolver, we do not include the fee for creating a change output as BnB will not create a change output. std::vector<OutputGroup> positive_groups = GroupOutputs(wallet, coins, coin_selection_params, eligibility_filter, true /* positive_only */); if (auto bnb_result{SelectCoinsBnB(positive_groups, nTargetValue, coin_selection_params.m_cost_of_change)}) { - bnb_result->ComputeAndSetWaste(CAmount(0)); results.push_back(*bnb_result); } @@ -387,15 +392,18 @@ std::optional<SelectionResult> AttemptSelection(const CWallet& wallet, const CAm std::vector<OutputGroup> all_groups = GroupOutputs(wallet, coins, coin_selection_params, eligibility_filter, false /* positive_only */); // While nTargetValue includes the transaction fees for non-input things, it does not include the fee for creating a change output. // So we need to include that for KnapsackSolver as well, as we are expecting to create a change output. - if (auto knapsack_result{KnapsackSolver(all_groups, nTargetValue + coin_selection_params.m_change_fee)}) { + if (auto knapsack_result{KnapsackSolver(all_groups, nTargetValue + coin_selection_params.m_change_fee, + coin_selection_params.m_min_change_target, coin_selection_params.rng_fast)}) { knapsack_result->ComputeAndSetWaste(coin_selection_params.m_cost_of_change); results.push_back(*knapsack_result); } - // We include the minimum final change for SRD as we do want to avoid making really small change. - // KnapsackSolver does not need this because it includes MIN_CHANGE internally. - const CAmount srd_target = nTargetValue + coin_selection_params.m_change_fee + MIN_FINAL_CHANGE; - if (auto srd_result{SelectCoinsSRD(positive_groups, srd_target)}) { + // Include change for SRD as we want to avoid making really small change if the selection just + // barely meets the target. Just use the lower bound change target instead of the randomly + // generated one, since SRD will result in a random change amount anyway; avoid making the + // target needlessly large. + const CAmount srd_target = nTargetValue + coin_selection_params.m_change_fee + CHANGE_LOWER; + if (auto srd_result{SelectCoinsSRD(positive_groups, srd_target, coin_selection_params.rng_fast)}) { srd_result->ComputeAndSetWaste(coin_selection_params.m_cost_of_change); results.push_back(*srd_result); } @@ -422,20 +430,21 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, const std::vec if (coin_control.HasSelected() && !coin_control.fAllowOtherInputs) { for (const COutput& out : vCoins) { - if (!out.fSpendable) continue; - /* Set depth, from_me, ancestors, and descendants to 0 or false as these don't matter for preset inputs as no actual selection is being done. + if (!out.spendable) continue; + /* Set ancestors and descendants to 0 as these don't matter for preset inputs as no actual selection is being done. * positive_only is set to false because we want to include all preset inputs, even if they are dust. */ - preset_inputs.Insert(out.GetInputCoin(), 0, false, 0, 0, false); + preset_inputs.Insert(out, /*ancestors=*/ 0, /*descendants=*/ 0, /*positive_only=*/ false); } - SelectionResult result(nTargetValue); + SelectionResult result(nTargetValue, SelectionAlgorithm::MANUAL); result.AddInput(preset_inputs); if (result.GetSelectedValue() < nTargetValue) return std::nullopt; + result.ComputeAndSetWaste(coin_selection_params.m_cost_of_change); return result; } // calculate value from preset inputs and store them - std::set<CInputCoin> setPresetCoins; + std::set<COutPoint> preset_coins; std::vector<COutPoint> vPresetInputs; coin_control.ListSelected(vPresetInputs); @@ -456,34 +465,36 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, const std::vec if (!coin_control.GetExternalOutput(outpoint, txout)) { return std::nullopt; } - input_bytes = CalculateMaximumSignedInputSize(txout, &coin_control.m_external_provider, /* use_max_sig */ true); + input_bytes = CalculateMaximumSignedInputSize(txout, &coin_control.m_external_provider, /*use_max_sig=*/true); } // If available, override calculated size with coin control specified size if (coin_control.HasInputWeight(outpoint)) { input_bytes = GetVirtualTransactionSize(coin_control.GetInputWeight(outpoint), 0, 0); } - CInputCoin coin(outpoint, txout, input_bytes); - if (coin.m_input_bytes == -1) { + if (input_bytes == -1) { return std::nullopt; // Not solvable, can't estimate size for fee } - coin.effective_value = coin.txout.nValue - coin_selection_params.m_effective_feerate.GetFee(coin.m_input_bytes); + + /* Set some defaults for depth, spendable, solvable, safe, time, and from_me as these don't matter for preset inputs since no selection is being done. */ + COutput output(outpoint, txout, /*depth=*/ 0, input_bytes, /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, /*time=*/ 0, /*from_me=*/ false); + output.effective_value = output.txout.nValue - coin_selection_params.m_effective_feerate.GetFee(output.input_bytes); if (coin_selection_params.m_subtract_fee_outputs) { - value_to_select -= coin.txout.nValue; + value_to_select -= output.txout.nValue; } else { - value_to_select -= coin.effective_value; + value_to_select -= output.effective_value; } - setPresetCoins.insert(coin); - /* Set depth, from_me, ancestors, and descendants to 0 or false as don't matter for preset inputs as no actual selection is being done. + preset_coins.insert(outpoint); + /* Set ancestors and descendants to 0 as they don't matter for preset inputs since no actual selection is being done. * positive_only is set to false because we want to include all preset inputs, even if they are dust. */ - preset_inputs.Insert(coin, 0, false, 0, 0, false); + preset_inputs.Insert(output, /*ancestors=*/ 0, /*descendants=*/ 0, /*positive_only=*/ false); } // remove preset inputs from vCoins so that Coin Selection doesn't pick them. for (std::vector<COutput>::iterator it = vCoins.begin(); it != vCoins.end() && coin_control.HasSelected();) { - if (setPresetCoins.count(it->GetInputCoin())) + if (preset_coins.count(it->outpoint)) it = vCoins.erase(it); else ++it; @@ -502,7 +513,7 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, const std::vec // Cases where we have 101+ outputs all pointing to the same destination may result in // privacy leaks as they will potentially be deterministically sorted. We solve that by // explicitly shuffling the outputs before processing - Shuffle(vCoins.begin(), vCoins.end(), FastRandomContext()); + Shuffle(vCoins.begin(), vCoins.end(), coin_selection_params.rng_fast); } // Coin Selection attempts to select inputs from a pool of eligible UTXOs to fund the @@ -510,7 +521,7 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, const std::vec // permissive CoinEligibilityFilter. std::optional<SelectionResult> res = [&] { // Pre-selected inputs already cover the target amount. - if (value_to_select <= 0) return std::make_optional(SelectionResult(nTargetValue)); + if (value_to_select <= 0) return std::make_optional(SelectionResult(nTargetValue, SelectionAlgorithm::MANUAL)); // If possible, fund the transaction with confirmed UTXOs only. Prefer at least six // confirmations on outputs received from other wallets and only spend confirmed change. @@ -564,6 +575,9 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, const std::vec // Add preset inputs to result res->AddInput(preset_inputs); + if (res->m_algo == SelectionAlgorithm::MANUAL) { + res->ComputeAndSetWaste(coin_selection_params.m_cost_of_change); + } return res; } @@ -583,12 +597,14 @@ static bool IsCurrentForAntiFeeSniping(interfaces::Chain& chain, const uint256& } /** - * Return a height-based locktime for new transactions (uses the height of the + * Set a height-based locktime for new transactions (uses the height of the * current chain tip unless we are not synced with the current chain */ -static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, const uint256& block_hash, int block_height) +static void DiscourageFeeSniping(CMutableTransaction& tx, FastRandomContext& rng_fast, + interfaces::Chain& chain, const uint256& block_hash, int block_height) { - uint32_t locktime; + // All inputs must be added by now + assert(!tx.vin.empty()); // Discourage fee sniping. // // For a large miner the value of the transactions in the best block and @@ -610,22 +626,34 @@ static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, const uin // now we ensure code won't be written that makes assumptions about // nLockTime that preclude a fix later. if (IsCurrentForAntiFeeSniping(chain, block_hash)) { - locktime = block_height; + tx.nLockTime = block_height; // Secondly occasionally randomly pick a nLockTime even further back, so // that transactions that are delayed after signing for whatever reason, // e.g. high-latency mix networks and some CoinJoin implementations, have // better privacy. - if (GetRandInt(10) == 0) - locktime = std::max(0, (int)locktime - GetRandInt(100)); + if (rng_fast.randrange(10) == 0) { + tx.nLockTime = std::max(0, int(tx.nLockTime) - int(rng_fast.randrange(100))); + } } else { // If our chain is lagging behind, we can't discourage fee sniping nor help // the privacy of high-latency transactions. To avoid leaking a potentially // unique "nLockTime fingerprint", set nLockTime to a constant. - locktime = 0; + tx.nLockTime = 0; + } + // Sanity check all values + assert(tx.nLockTime < LOCKTIME_THRESHOLD); // Type must be block height + assert(tx.nLockTime <= uint64_t(block_height)); + for (const auto& in : tx.vin) { + // Can not be FINAL for locktime to work + assert(in.nSequence != CTxIn::SEQUENCE_FINAL); + // May be MAX NONFINAL to disable both BIP68 and BIP125 + if (in.nSequence == CTxIn::MAX_SEQUENCE_NONFINAL) continue; + // May be MAX BIP125 to disable BIP68 and enable BIP125 + if (in.nSequence == MAX_BIP125_RBF_SEQUENCE) continue; + // The wallet does not support any other sequence-use right now. + assert(false); } - assert(locktime < LOCKTIME_THRESHOLD); - return locktime; } static bool CreateTransactionInternal( @@ -641,10 +669,10 @@ static bool CreateTransactionInternal( { AssertLockHeld(wallet.cs_wallet); + FastRandomContext rng_fast; CMutableTransaction txNew; // The resulting transaction that we make - txNew.nLockTime = GetLocktimeForNewTransaction(wallet.chain(), wallet.GetLastBlockHash(), wallet.GetLastBlockHeight()); - CoinSelectionParams coin_selection_params; // Parameters for coin selection, init with dummy + CoinSelectionParams coin_selection_params{rng_fast}; // Parameters for coin selection, init with dummy coin_selection_params.m_avoid_partial_spends = coin_control.m_avoid_partial_spends; // Set the long term feerate estimate to the wallet's consolidate feerate @@ -662,6 +690,7 @@ static bool CreateTransactionInternal( coin_selection_params.m_subtract_fee_outputs = true; } } + coin_selection_params.m_change_target = GenerateChangeTarget(std::floor(recipients_sum / vecSend.size()), rng_fast); // Create change script that will be used if we need change CScript scriptChange; @@ -759,11 +788,12 @@ static bool CreateTransactionInternal( AvailableCoins(wallet, vAvailableCoins, &coin_control, 1, MAX_MONEY, MAX_MONEY, 0); // Choose coins to use - std::optional<SelectionResult> result = SelectCoins(wallet, vAvailableCoins, /* nTargetValue */ selection_target, coin_control, coin_selection_params); + std::optional<SelectionResult> result = SelectCoins(wallet, vAvailableCoins, /*nTargetValue=*/selection_target, coin_control, coin_selection_params); if (!result) { error = _("Insufficient funds"); return false; } + TRACE5(coin_selection, selected_coins, wallet.GetName().c_str(), GetAlgorithmName(result->m_algo).c_str(), result->m_target, result->GetWaste(), result->GetSelectedValue()); // Always make a change output // We will reduce the fee from this change output later, and remove the output if it is too small. @@ -771,10 +801,9 @@ static bool CreateTransactionInternal( assert(change_and_fee >= 0); CTxOut newTxOut(change_and_fee, scriptChange); - if (nChangePosInOut == -1) - { + if (nChangePosInOut == -1) { // Insert change txn at random position: - nChangePosInOut = GetRandInt(txNew.vout.size()+1); + nChangePosInOut = rng_fast.randrange(txNew.vout.size() + 1); } else if ((unsigned int)nChangePosInOut > txNew.vout.size()) { @@ -786,10 +815,10 @@ static bool CreateTransactionInternal( auto change_position = txNew.vout.insert(txNew.vout.begin() + nChangePosInOut, newTxOut); // Shuffle selected coins and fill in final vin - std::vector<CInputCoin> selected_coins = result->GetShuffledInputVector(); + std::vector<COutput> selected_coins = result->GetShuffledInputVector(); - // Note how the sequence number is set to non-maxint so that - // the nLockTime set above actually works. + // The sequence number is set to non-maxint so that DiscourageFeeSniping + // works. // // BIP125 defines opt-in RBF as any nSequence < maxint-1, so // we use the highest possible value in that range (maxint-2) @@ -800,6 +829,7 @@ static bool CreateTransactionInternal( for (const auto& coin : selected_coins) { txNew.vin.push_back(CTxIn(coin.outpoint, CScript(), nSequence)); } + DiscourageFeeSniping(txNew, rng_fast, wallet.chain(), wallet.GetLastBlockHash(), wallet.GetLastBlockHeight()); // Calculate the transaction fee TxSize tx_sizes = CalculateMaximumSignedTxSize(CTransaction(txNew), &wallet, &coin_control); @@ -954,8 +984,10 @@ bool CreateTransaction( int nChangePosIn = nChangePosInOut; Assert(!tx); // tx is an out-param. TODO change the return type from bool to tx (or nullptr) bool res = CreateTransactionInternal(wallet, vecSend, tx, nFeeRet, nChangePosInOut, error, coin_control, fee_calc_out, sign); + TRACE4(coin_selection, normal_create_tx_internal, wallet.GetName().c_str(), res, nFeeRet, nChangePosInOut); // try with avoidpartialspends unless it's enabled already if (res && nFeeRet > 0 /* 0 means non-functional fee rate estimation */ && wallet.m_max_aps_fee > -1 && !coin_control.m_avoid_partial_spends) { + TRACE1(coin_selection, attempting_aps_create_tx, wallet.GetName().c_str()); CCoinControl tmp_cc = coin_control; tmp_cc.m_avoid_partial_spends = true; CAmount nFeeRet2; @@ -966,6 +998,7 @@ bool CreateTransaction( // if fee of this alternative one is within the range of the max fee, we use this one const bool use_aps = nFeeRet2 <= nFeeRet + wallet.m_max_aps_fee; wallet.WalletLogPrintf("Fee non-grouped = %lld, grouped = %lld, using %s\n", nFeeRet, nFeeRet2, use_aps ? "grouped" : "non-grouped"); + TRACE5(coin_selection, aps_create_tx_internal, wallet.GetName().c_str(), use_aps, res, nFeeRet2, nChangePosInOut2); if (use_aps) { tx = tx2; nFeeRet = nFeeRet2; diff --git a/src/wallet/spend.h b/src/wallet/spend.h index 4453fb2762..e43aac5273 100644 --- a/src/wallet/spend.h +++ b/src/wallet/spend.h @@ -11,61 +11,11 @@ #include <wallet/wallet.h> namespace wallet { -/** Get the marginal bytes if spending the specified output from this transaction */ +/** Get the marginal bytes if spending the specified output from this transaction. + * use_max_sig indicates whether to use the maximum sized, 72 byte signature when calculating the + * size of the input spend. This should only be set when watch-only outputs are allowed */ int GetTxSpendSize(const CWallet& wallet, const CWalletTx& wtx, unsigned int out, bool use_max_sig = false); -class COutput -{ -public: - const CWalletTx *tx; - - /** Index in tx->vout. */ - int i; - - /** - * Depth in block chain. - * If > 0: the tx is on chain and has this many confirmations. - * If = 0: the tx is waiting confirmation. - * If < 0: a conflicting tx is on chain and has this many confirmations. */ - int nDepth; - - /** Pre-computed estimated size of this output as a fully-signed input in a transaction. Can be -1 if it could not be calculated */ - int nInputBytes; - - /** Whether we have the private keys to spend this output */ - bool fSpendable; - - /** Whether we know how to spend this output, ignoring the lack of keys */ - bool fSolvable; - - /** Whether to use the maximum sized, 72 byte signature when calculating the size of the input spend. This should only be set when watch-only outputs are allowed */ - bool use_max_sig; - - /** - * Whether this output is considered safe to spend. Unconfirmed transactions - * from outside keys and unconfirmed replacement transactions are considered - * unsafe and will not be used to fund new spending transactions. - */ - bool fSafe; - - COutput(const CWallet& wallet, const CWalletTx& wtx, int iIn, int nDepthIn, bool fSpendableIn, bool fSolvableIn, bool fSafeIn, bool use_max_sig_in = false) - { - tx = &wtx; i = iIn; nDepth = nDepthIn; fSpendable = fSpendableIn; fSolvable = fSolvableIn; fSafe = fSafeIn; nInputBytes = -1; use_max_sig = use_max_sig_in; - // If known and signable by the given wallet, compute nInputBytes - // Failure will keep this value -1 - if (fSpendable) { - nInputBytes = GetTxSpendSize(wallet, wtx, i, use_max_sig); - } - } - - std::string ToString() const; - - inline CInputCoin GetInputCoin() const - { - return CInputCoin(tx->tx, i, nInputBytes); - } -}; - //Get the marginal bytes of spending the specified output int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* pwallet, bool use_max_sig = false); int CalculateMaximumSignedInputSize(const CTxOut& txout, const SigningProvider* pwallet, bool use_max_sig = false); @@ -93,6 +43,7 @@ CAmount GetAvailableBalance(const CWallet& wallet, const CCoinControl* coinContr * Find non-change parent output. */ const CTxOut& FindNonChangeParentOutput(const CWallet& wallet, const CTransaction& tx, int output) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); +const CTxOut& FindNonChangeParentOutput(const CWallet& wallet, const COutPoint& outpoint) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); /** * Return list of available coins and locked coins grouped by non-change output address. diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index 2b2181e70b..2515df3177 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -37,6 +37,22 @@ static void ErrorLogCallback(void* arg, int code, const char* msg) LogPrintf("SQLite Error. Code: %d. Message: %s\n", code, msg); } +static bool BindBlobToStatement(sqlite3_stmt* stmt, + int index, + Span<const std::byte> blob, + const std::string& description) +{ + int res = sqlite3_bind_blob(stmt, index, blob.data(), blob.size(), SQLITE_STATIC); + if (res != SQLITE_OK) { + LogPrintf("Unable to bind %s to statement: %s\n", description, sqlite3_errstr(res)); + sqlite3_clear_bindings(stmt); + sqlite3_reset(stmt); + return false; + } + + return true; +} + static std::optional<int> ReadPragmaInteger(sqlite3* db, const std::string& key, const std::string& description, bilingual_str& error) { std::string stmt_text = strprintf("PRAGMA %s", key); @@ -67,8 +83,8 @@ static void SetPragma(sqlite3* db, const std::string& key, const std::string& va } } -SQLiteDatabase::SQLiteDatabase(const fs::path& dir_path, const fs::path& file_path, bool mock) - : WalletDatabase(), m_mock(mock), m_dir_path(fs::PathToString(dir_path)), m_file_path(fs::PathToString(file_path)) +SQLiteDatabase::SQLiteDatabase(const fs::path& dir_path, const fs::path& file_path, const DatabaseOptions& options, bool mock) + : WalletDatabase(), m_mock(mock), m_dir_path(fs::PathToString(dir_path)), m_file_path(fs::PathToString(file_path)), m_use_unsafe_sync(options.use_unsafe_sync) { { LOCK(g_sqlite_mutex); @@ -239,7 +255,7 @@ void SQLiteDatabase::Open() // Enable fullfsync for the platforms that use it SetPragma(m_db, "fullfsync", "true", "Failed to enable fullfsync"); - if (gArgs.GetBoolArg("-unsafesqlitesync", false)) { + if (m_use_unsafe_sync) { // Use normal synchronous mode for the journal LogPrintf("WARNING SQLite is configured to not wait for data to be flushed to disk. Data loss and corruption may occur.\n"); SetPragma(m_db, "synchronous", "OFF", "Failed to set synchronous mode to OFF"); @@ -377,14 +393,8 @@ bool SQLiteBatch::ReadKey(CDataStream&& key, CDataStream& value) assert(m_read_stmt); // Bind: leftmost parameter in statement is index 1 - int res = sqlite3_bind_blob(m_read_stmt, 1, key.data(), key.size(), SQLITE_STATIC); - if (res != SQLITE_OK) { - LogPrintf("%s: Unable to bind statement: %s\n", __func__, sqlite3_errstr(res)); - sqlite3_clear_bindings(m_read_stmt); - sqlite3_reset(m_read_stmt); - return false; - } - res = sqlite3_step(m_read_stmt); + if (!BindBlobToStatement(m_read_stmt, 1, key, "key")) return false; + int res = sqlite3_step(m_read_stmt); if (res != SQLITE_ROW) { if (res != SQLITE_DONE) { // SQLITE_DONE means "not found", don't log an error in that case. @@ -395,7 +405,7 @@ bool SQLiteBatch::ReadKey(CDataStream&& key, CDataStream& value) return false; } // Leftmost column in result is index 0 - const std::byte* data{BytePtr(sqlite3_column_blob(m_read_stmt, 0))}; + const std::byte* data{AsBytePtr(sqlite3_column_blob(m_read_stmt, 0))}; size_t data_size(sqlite3_column_bytes(m_read_stmt, 0)); value.write({data, data_size}); @@ -418,23 +428,11 @@ bool SQLiteBatch::WriteKey(CDataStream&& key, CDataStream&& value, bool overwrit // Bind: leftmost parameter in statement is index 1 // Insert index 1 is key, 2 is value - int res = sqlite3_bind_blob(stmt, 1, key.data(), key.size(), SQLITE_STATIC); - if (res != SQLITE_OK) { - LogPrintf("%s: Unable to bind key to statement: %s\n", __func__, sqlite3_errstr(res)); - sqlite3_clear_bindings(stmt); - sqlite3_reset(stmt); - return false; - } - res = sqlite3_bind_blob(stmt, 2, value.data(), value.size(), SQLITE_STATIC); - if (res != SQLITE_OK) { - LogPrintf("%s: Unable to bind value to statement: %s\n", __func__, sqlite3_errstr(res)); - sqlite3_clear_bindings(stmt); - sqlite3_reset(stmt); - return false; - } + if (!BindBlobToStatement(stmt, 1, key, "key")) return false; + if (!BindBlobToStatement(stmt, 2, value, "value")) return false; // Execute - res = sqlite3_step(stmt); + int res = sqlite3_step(stmt); sqlite3_clear_bindings(stmt); sqlite3_reset(stmt); if (res != SQLITE_DONE) { @@ -449,16 +447,10 @@ bool SQLiteBatch::EraseKey(CDataStream&& key) assert(m_delete_stmt); // Bind: leftmost parameter in statement is index 1 - int res = sqlite3_bind_blob(m_delete_stmt, 1, key.data(), key.size(), SQLITE_STATIC); - if (res != SQLITE_OK) { - LogPrintf("%s: Unable to bind statement: %s\n", __func__, sqlite3_errstr(res)); - sqlite3_clear_bindings(m_delete_stmt); - sqlite3_reset(m_delete_stmt); - return false; - } + if (!BindBlobToStatement(m_delete_stmt, 1, key, "key")) return false; // Execute - res = sqlite3_step(m_delete_stmt); + int res = sqlite3_step(m_delete_stmt); sqlite3_clear_bindings(m_delete_stmt); sqlite3_reset(m_delete_stmt); if (res != SQLITE_DONE) { @@ -473,18 +465,11 @@ bool SQLiteBatch::HasKey(CDataStream&& key) assert(m_read_stmt); // Bind: leftmost parameter in statement is index 1 - bool ret = false; - int res = sqlite3_bind_blob(m_read_stmt, 1, key.data(), key.size(), SQLITE_STATIC); - if (res == SQLITE_OK) { - res = sqlite3_step(m_read_stmt); - if (res == SQLITE_ROW) { - ret = true; - } - } - + if (!BindBlobToStatement(m_read_stmt, 1, key, "key")) return false; + int res = sqlite3_step(m_read_stmt); sqlite3_clear_bindings(m_read_stmt); sqlite3_reset(m_read_stmt); - return ret; + return res == SQLITE_ROW; } bool SQLiteBatch::StartCursor() @@ -512,10 +497,10 @@ bool SQLiteBatch::ReadAtCursor(CDataStream& key, CDataStream& value, bool& compl } // Leftmost column in result is index 0 - const std::byte* key_data{BytePtr(sqlite3_column_blob(m_cursor_stmt, 0))}; + const std::byte* key_data{AsBytePtr(sqlite3_column_blob(m_cursor_stmt, 0))}; size_t key_data_size(sqlite3_column_bytes(m_cursor_stmt, 0)); key.write({key_data, key_data_size}); - const std::byte* value_data{BytePtr(sqlite3_column_blob(m_cursor_stmt, 1))}; + const std::byte* value_data{AsBytePtr(sqlite3_column_blob(m_cursor_stmt, 1))}; size_t value_data_size(sqlite3_column_bytes(m_cursor_stmt, 1)); value.write({value_data, value_data_size}); return true; @@ -561,7 +546,7 @@ std::unique_ptr<SQLiteDatabase> MakeSQLiteDatabase(const fs::path& path, const D { try { fs::path data_file = SQLiteDataFile(path); - auto db = std::make_unique<SQLiteDatabase>(data_file.parent_path(), data_file); + auto db = std::make_unique<SQLiteDatabase>(data_file.parent_path(), data_file, options); if (options.verify && !db->Verify(error)) { status = DatabaseStatus::FAILED_VERIFY; return nullptr; diff --git a/src/wallet/sqlite.h b/src/wallet/sqlite.h index 3ed598d0d2..47b7ebb0ec 100644 --- a/src/wallet/sqlite.h +++ b/src/wallet/sqlite.h @@ -69,7 +69,7 @@ public: SQLiteDatabase() = delete; /** Create DB handle to real database */ - SQLiteDatabase(const fs::path& dir_path, const fs::path& file_path, bool mock = false); + SQLiteDatabase(const fs::path& dir_path, const fs::path& file_path, const DatabaseOptions& options, bool mock = false); ~SQLiteDatabase(); @@ -113,6 +113,7 @@ public: std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) override; sqlite3* m_db{nullptr}; + bool m_use_unsafe_sync; }; std::unique_ptr<SQLiteDatabase> MakeSQLiteDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index b9f12158ca..72e749477b 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -28,20 +28,20 @@ BOOST_FIXTURE_TEST_SUITE(coinselector_tests, WalletTestingSetup) // we repeat those tests this many times and only complain if all iterations of the test fail #define RANDOM_REPEATS 5 -typedef std::set<CInputCoin> CoinSet; +typedef std::set<COutput> CoinSet; static const CoinEligibilityFilter filter_standard(1, 6, 0); static const CoinEligibilityFilter filter_confirmed(1, 1, 0); static const CoinEligibilityFilter filter_standard_extra(6, 6, 0); static int nextLockTime = 0; -static void add_coin(const CAmount& nValue, int nInput, std::vector<CInputCoin>& set) +static void add_coin(const CAmount& nValue, int nInput, std::vector<COutput>& set) { CMutableTransaction tx; tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; tx.nLockTime = nextLockTime++; // so all transactions get different hashes - set.emplace_back(MakeTransactionRef(tx), nInput); + set.emplace_back(COutPoint(tx.GetHash(), nInput), tx.vout.at(nInput), /*depth=*/ 1, /*input_bytes=*/ -1, /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, /*time=*/ 0, /*from_me=*/ false); } static void add_coin(const CAmount& nValue, int nInput, SelectionResult& result) @@ -50,9 +50,9 @@ static void add_coin(const CAmount& nValue, int nInput, SelectionResult& result) tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; tx.nLockTime = nextLockTime++; // so all transactions get different hashes - CInputCoin coin(MakeTransactionRef(tx), nInput); + COutput output(COutPoint(tx.GetHash(), nInput), tx.vout.at(nInput), /*depth=*/ 1, /*input_bytes=*/ -1, /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, /*time=*/ 0, /*from_me=*/ false); OutputGroup group; - group.Insert(coin, 1, false, 0, 0, true); + group.Insert(output, /*ancestors=*/ 0, /*descendants=*/ 0, /*positive_only=*/ true); result.AddInput(group); } @@ -62,10 +62,10 @@ static void add_coin(const CAmount& nValue, int nInput, CoinSet& set, CAmount fe tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; tx.nLockTime = nextLockTime++; // so all transactions get different hashes - CInputCoin coin(MakeTransactionRef(tx), nInput); + COutput coin(COutPoint(tx.GetHash(), nInput), tx.vout.at(nInput), /*depth=*/ 1, /*input_bytes=*/ -1, /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, /*time=*/ 0, /*from_me=*/ false); coin.effective_value = nValue - fee; - coin.m_fee = fee; - coin.m_long_term_fee = long_term_fee; + coin.fee = fee; + coin.long_term_fee = long_term_fee; set.insert(coin); } @@ -82,24 +82,13 @@ static void add_coin(std::vector<COutput>& coins, CWallet& wallet, const CAmount assert(destination_ok); tx.vout[nInput].scriptPubKey = GetScriptForDestination(dest); } - if (fIsFromMe) { - // IsFromMe() returns (GetDebit() > 0), and GetDebit() is 0 if vin.empty(), - // so stop vin being empty, and cache a non-zero Debit to fake out IsFromMe() - tx.vin.resize(1); - } uint256 txid = tx.GetHash(); LOCK(wallet.cs_wallet); auto ret = wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(txid), std::forward_as_tuple(MakeTransactionRef(std::move(tx)), TxStateInactive{})); assert(ret.second); CWalletTx& wtx = (*ret.first).second; - if (fIsFromMe) - { - wtx.m_amounts[CWalletTx::DEBIT].Set(ISMINE_SPENDABLE, 1); - wtx.m_is_cache_empty = false; - } - COutput output(wallet, wtx, nInput, nAge, true /* spendable */, true /* solvable */, true /* safe */); - coins.push_back(output); + coins.emplace_back(COutPoint(wtx.GetHash(), nInput), wtx.tx->vout.at(nInput), nAge, GetTxSpendSize(wallet, wtx, nInput), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, wtx.GetTxTime(), fIsFromMe); } /** Check if SelectionResult a is equivalent to SelectionResult b. @@ -124,11 +113,14 @@ static bool EquivalentResult(const SelectionResult& a, const SelectionResult& b) /** Check if this selection is equal to another one. Equal means same inputs (i.e same value and prevout) */ static bool EqualResult(const SelectionResult& a, const SelectionResult& b) { - std::pair<CoinSet::iterator, CoinSet::iterator> ret = std::mismatch(a.GetInputSet().begin(), a.GetInputSet().end(), b.GetInputSet().begin()); + std::pair<CoinSet::iterator, CoinSet::iterator> ret = std::mismatch(a.GetInputSet().begin(), a.GetInputSet().end(), b.GetInputSet().begin(), + [](const COutput& a, const COutput& b) { + return a.outpoint == b.outpoint; + }); return ret.first == a.GetInputSet().end() && ret.second == b.GetInputSet().end(); } -static CAmount make_hard_case(int utxos, std::vector<CInputCoin>& utxo_pool) +static CAmount make_hard_case(int utxos, std::vector<COutput>& utxo_pool) { utxo_pool.clear(); CAmount target = 0; @@ -140,34 +132,31 @@ static CAmount make_hard_case(int utxos, std::vector<CInputCoin>& utxo_pool) return target; } -inline std::vector<OutputGroup>& GroupCoins(const std::vector<CInputCoin>& coins) -{ - static std::vector<OutputGroup> static_groups; - static_groups.clear(); - for (auto& coin : coins) { - static_groups.emplace_back(); - static_groups.back().Insert(coin, 0, true, 0, 0, false); - } - return static_groups; -} - inline std::vector<OutputGroup>& GroupCoins(const std::vector<COutput>& coins) { static std::vector<OutputGroup> static_groups; static_groups.clear(); for (auto& coin : coins) { static_groups.emplace_back(); - static_groups.back().Insert(coin.GetInputCoin(), coin.nDepth, coin.tx->m_amounts[CWalletTx::DEBIT].m_cached[ISMINE_SPENDABLE] && coin.tx->m_amounts[CWalletTx::DEBIT].m_value[ISMINE_SPENDABLE] == 1 /* HACK: we can't figure out the is_me flag so we use the conditions defined above; perhaps set safe to false for !fIsFromMe in add_coin() */, 0, 0, false); + static_groups.back().Insert(coin, /*ancestors=*/ 0, /*descendants=*/ 0, /*positive_only=*/ false); } return static_groups; } inline std::vector<OutputGroup>& KnapsackGroupOutputs(const std::vector<COutput>& coins, CWallet& wallet, const CoinEligibilityFilter& filter) { - CoinSelectionParams coin_selection_params(/* change_output_size= */ 0, - /* change_spend_size= */ 0, /* effective_feerate= */ CFeeRate(0), - /* long_term_feerate= */ CFeeRate(0), /* discard_feerate= */ CFeeRate(0), - /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); + FastRandomContext rand{}; + CoinSelectionParams coin_selection_params{ + rand, + /*change_output_size=*/ 0, + /*change_spend_size=*/ 0, + /*min_change_target=*/ CENT, + /*effective_feerate=*/ CFeeRate(0), + /*long_term_feerate=*/ CFeeRate(0), + /*discard_feerate=*/ CFeeRate(0), + /*tx_noinputs_size=*/ 0, + /*avoid_partial=*/ false, + }; static std::vector<OutputGroup> static_groups; static_groups = GroupOutputs(wallet, coins, coin_selection_params, filter, /*positive_only=*/false); return static_groups; @@ -176,9 +165,10 @@ inline std::vector<OutputGroup>& KnapsackGroupOutputs(const std::vector<COutput> // Branch and bound coin selection tests BOOST_AUTO_TEST_CASE(bnb_search_test) { + FastRandomContext rand{}; // Setup - std::vector<CInputCoin> utxo_pool; - SelectionResult expected_result(CAmount(0)); + std::vector<COutput> utxo_pool; + SelectionResult expected_result(CAmount(0), SelectionAlgorithm::BNB); ///////////////////////// // Known Outcome tests // @@ -301,10 +291,17 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) } // Make sure that effective value is working in AttemptSelection when BnB is used - CoinSelectionParams coin_selection_params_bnb(/* change_output_size= */ 0, - /* change_spend_size= */ 0, /* effective_feerate= */ CFeeRate(3000), - /* long_term_feerate= */ CFeeRate(1000), /* discard_feerate= */ CFeeRate(1000), - /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); + CoinSelectionParams coin_selection_params_bnb{ + rand, + /*change_output_size=*/ 0, + /*change_spend_size=*/ 0, + /*min_change_target=*/ 0, + /*effective_feerate=*/ CFeeRate(3000), + /*long_term_feerate=*/ CFeeRate(1000), + /*discard_feerate=*/ CFeeRate(1000), + /*tx_noinputs_size=*/ 0, + /*avoid_partial=*/ false, + }; { std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()); wallet->LoadWallet(); @@ -315,13 +312,13 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) std::vector<COutput> coins; add_coin(coins, *wallet, 1); - coins.at(0).nInputBytes = 40; // Make sure that it has a negative effective value. The next check should assert if this somehow got through. Otherwise it will fail + coins.at(0).input_bytes = 40; // Make sure that it has a negative effective value. The next check should assert if this somehow got through. Otherwise it will fail BOOST_CHECK(!SelectCoinsBnB(GroupCoins(coins), 1 * CENT, coin_selection_params_bnb.m_cost_of_change)); // Test fees subtracted from output: coins.clear(); add_coin(coins, *wallet, 1 * CENT); - coins.at(0).nInputBytes = 40; + coins.at(0).input_bytes = 40; coin_selection_params_bnb.m_subtract_fee_outputs = true; const auto result9 = SelectCoinsBnB(GroupCoins(coins), 1 * CENT, coin_selection_params_bnb.m_cost_of_change); BOOST_CHECK(result9); @@ -342,7 +339,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) add_coin(coins, *wallet, 2 * CENT, 6 * 24, false, 0, true); CCoinControl coin_control; coin_control.fAllowOtherInputs = true; - coin_control.Select(COutPoint(coins.at(0).tx->GetHash(), coins.at(0).i)); + coin_control.Select(coins.at(0).outpoint); coin_selection_params_bnb.m_effective_feerate = CFeeRate(0); const auto result10 = SelectCoins(*wallet, coins, 10 * CENT, coin_control, coin_selection_params_bnb); BOOST_CHECK(result10); @@ -351,6 +348,9 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) BOOST_AUTO_TEST_CASE(knapsack_solver_test) { + FastRandomContext rand{}; + const auto temp1{[&rand](std::vector<OutputGroup>& g, const CAmount& v, CAmount c) { return KnapsackSolver(g, v, c, rand); }}; + const auto KnapsackSolver{temp1}; std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()); wallet->LoadWallet(); LOCK(wallet->cs_wallet); @@ -365,25 +365,25 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) coins.clear(); // with an empty wallet we can't even pay one cent - BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 1 * CENT)); + BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 1 * CENT, CENT)); add_coin(coins, *wallet, 1*CENT, 4); // add a new 1 cent coin // with a new 1 cent coin, we still can't find a mature 1 cent - BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 1 * CENT)); + BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 1 * CENT, CENT)); // but we can find a new 1 cent - const auto result1 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 1 * CENT); + const auto result1 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 1 * CENT, CENT); BOOST_CHECK(result1); BOOST_CHECK_EQUAL(result1->GetSelectedValue(), 1 * CENT); add_coin(coins, *wallet, 2*CENT); // add a mature 2 cent coin // we can't make 3 cents of mature coins - BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 3 * CENT)); + BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 3 * CENT, CENT)); // we can make 3 cents of new coins - const auto result2 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 3 * CENT); + const auto result2 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 3 * CENT, CENT); BOOST_CHECK(result2); BOOST_CHECK_EQUAL(result2->GetSelectedValue(), 3 * CENT); @@ -394,38 +394,38 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // now we have new: 1+10=11 (of which 10 was self-sent), and mature: 2+5+20=27. total = 38 // we can't make 38 cents only if we disallow new coins: - BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 38 * CENT)); + BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 38 * CENT, CENT)); // we can't even make 37 cents if we don't allow new coins even if they're from us - BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard_extra), 38 * CENT)); + BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard_extra), 38 * CENT, CENT)); // but we can make 37 cents if we accept new coins from ourself - const auto result3 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 37 * CENT); + const auto result3 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 37 * CENT, CENT); BOOST_CHECK(result3); BOOST_CHECK_EQUAL(result3->GetSelectedValue(), 37 * CENT); // and we can make 38 cents if we accept all new coins - const auto result4 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 38 * CENT); + const auto result4 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 38 * CENT, CENT); BOOST_CHECK(result4); BOOST_CHECK_EQUAL(result4->GetSelectedValue(), 38 * CENT); // try making 34 cents from 1,2,5,10,20 - we can't do it exactly - const auto result5 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 34 * CENT); + const auto result5 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 34 * CENT, CENT); BOOST_CHECK(result5); BOOST_CHECK_EQUAL(result5->GetSelectedValue(), 35 * CENT); // but 35 cents is closest BOOST_CHECK_EQUAL(result5->GetInputSet().size(), 3U); // the best should be 20+10+5. it's incredibly unlikely the 1 or 2 got included (but possible) // when we try making 7 cents, the smaller coins (1,2,5) are enough. We should see just 2+5 - const auto result6 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 7 * CENT); + const auto result6 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 7 * CENT, CENT); BOOST_CHECK(result6); BOOST_CHECK_EQUAL(result6->GetSelectedValue(), 7 * CENT); BOOST_CHECK_EQUAL(result6->GetInputSet().size(), 2U); // when we try making 8 cents, the smaller coins (1,2,5) are exactly enough. - const auto result7 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 8 * CENT); + const auto result7 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 8 * CENT, CENT); BOOST_CHECK(result7); BOOST_CHECK(result7->GetSelectedValue() == 8 * CENT); BOOST_CHECK_EQUAL(result7->GetInputSet().size(), 3U); // when we try making 9 cents, no subset of smaller coins is enough, and we get the next bigger coin (10) - const auto result8 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 9 * CENT); + const auto result8 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 9 * CENT, CENT); BOOST_CHECK(result8); BOOST_CHECK_EQUAL(result8->GetSelectedValue(), 10 * CENT); BOOST_CHECK_EQUAL(result8->GetInputSet().size(), 1U); @@ -440,12 +440,12 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(coins, *wallet, 30*CENT); // now we have 6+7+8+20+30 = 71 cents total // check that we have 71 and not 72 - const auto result9 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 71 * CENT); + const auto result9 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 71 * CENT, CENT); BOOST_CHECK(result9); - BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 72 * CENT)); + BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 72 * CENT, CENT)); // now try making 16 cents. the best smaller coins can do is 6+7+8 = 21; not as good at the next biggest coin, 20 - const auto result10 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 16 * CENT); + const auto result10 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 16 * CENT, CENT); BOOST_CHECK(result10); BOOST_CHECK_EQUAL(result10->GetSelectedValue(), 20 * CENT); // we should get 20 in one coin BOOST_CHECK_EQUAL(result10->GetInputSet().size(), 1U); @@ -453,7 +453,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(coins, *wallet, 5*CENT); // now we have 5+6+7+8+20+30 = 75 cents total // now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, better than the next biggest coin, 20 - const auto result11 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 16 * CENT); + const auto result11 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 16 * CENT, CENT); BOOST_CHECK(result11); BOOST_CHECK_EQUAL(result11->GetSelectedValue(), 18 * CENT); // we should get 18 in 3 coins BOOST_CHECK_EQUAL(result11->GetInputSet().size(), 3U); @@ -461,13 +461,13 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(coins, *wallet, 18*CENT); // now we have 5+6+7+8+18+20+30 // and now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, the same as the next biggest coin, 18 - const auto result12 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 16 * CENT); + const auto result12 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 16 * CENT, CENT); BOOST_CHECK(result12); BOOST_CHECK_EQUAL(result12->GetSelectedValue(), 18 * CENT); // we should get 18 in 1 coin BOOST_CHECK_EQUAL(result12->GetInputSet().size(), 1U); // because in the event of a tie, the biggest coin wins // now try making 11 cents. we should get 5+6 - const auto result13 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 11 * CENT); + const auto result13 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 11 * CENT, CENT); BOOST_CHECK(result13); BOOST_CHECK_EQUAL(result13->GetSelectedValue(), 11 * CENT); BOOST_CHECK_EQUAL(result13->GetInputSet().size(), 2U); @@ -477,12 +477,12 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(coins, *wallet, 2*COIN); add_coin(coins, *wallet, 3*COIN); add_coin(coins, *wallet, 4*COIN); // now we have 5+6+7+8+18+20+30+100+200+300+400 = 1094 cents - const auto result14 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 95 * CENT); + const auto result14 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 95 * CENT, CENT); BOOST_CHECK(result14); BOOST_CHECK_EQUAL(result14->GetSelectedValue(), 1 * COIN); // we should get 1 BTC in 1 coin BOOST_CHECK_EQUAL(result14->GetInputSet().size(), 1U); - const auto result15 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 195 * CENT); + const auto result15 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 195 * CENT, CENT); BOOST_CHECK(result15); BOOST_CHECK_EQUAL(result15->GetSelectedValue(), 2 * COIN); // we should get 2 BTC in 1 coin BOOST_CHECK_EQUAL(result15->GetInputSet().size(), 1U); @@ -490,34 +490,34 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // empty the wallet and start again, now with fractions of a cent, to test small change avoidance coins.clear(); - add_coin(coins, *wallet, MIN_CHANGE * 1 / 10); - add_coin(coins, *wallet, MIN_CHANGE * 2 / 10); - add_coin(coins, *wallet, MIN_CHANGE * 3 / 10); - add_coin(coins, *wallet, MIN_CHANGE * 4 / 10); - add_coin(coins, *wallet, MIN_CHANGE * 5 / 10); - - // try making 1 * MIN_CHANGE from the 1.5 * MIN_CHANGE - // we'll get change smaller than MIN_CHANGE whatever happens, so can expect MIN_CHANGE exactly - const auto result16 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), MIN_CHANGE); + add_coin(coins, *wallet, CENT * 1 / 10); + add_coin(coins, *wallet, CENT * 2 / 10); + add_coin(coins, *wallet, CENT * 3 / 10); + add_coin(coins, *wallet, CENT * 4 / 10); + add_coin(coins, *wallet, CENT * 5 / 10); + + // try making 1 * CENT from the 1.5 * CENT + // we'll get change smaller than CENT whatever happens, so can expect CENT exactly + const auto result16 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), CENT, CENT); BOOST_CHECK(result16); - BOOST_CHECK_EQUAL(result16->GetSelectedValue(), MIN_CHANGE); + BOOST_CHECK_EQUAL(result16->GetSelectedValue(), CENT); // but if we add a bigger coin, small change is avoided - add_coin(coins, *wallet, 1111*MIN_CHANGE); + add_coin(coins, *wallet, 1111*CENT); // try making 1 from 0.1 + 0.2 + 0.3 + 0.4 + 0.5 + 1111 = 1112.5 - const auto result17 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 1 * MIN_CHANGE); + const auto result17 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 1 * CENT, CENT); BOOST_CHECK(result17); - BOOST_CHECK_EQUAL(result17->GetSelectedValue(), 1 * MIN_CHANGE); // we should get the exact amount + BOOST_CHECK_EQUAL(result17->GetSelectedValue(), 1 * CENT); // we should get the exact amount // if we add more small coins: - add_coin(coins, *wallet, MIN_CHANGE * 6 / 10); - add_coin(coins, *wallet, MIN_CHANGE * 7 / 10); + add_coin(coins, *wallet, CENT * 6 / 10); + add_coin(coins, *wallet, CENT * 7 / 10); - // and try again to make 1.0 * MIN_CHANGE - const auto result18 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 1 * MIN_CHANGE); + // and try again to make 1.0 * CENT + const auto result18 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 1 * CENT, CENT); BOOST_CHECK(result18); - BOOST_CHECK_EQUAL(result18->GetSelectedValue(), 1 * MIN_CHANGE); // we should get the exact amount + BOOST_CHECK_EQUAL(result18->GetSelectedValue(), 1 * CENT); // we should get the exact amount // run the 'mtgox' test (see https://blockexplorer.com/tx/29a3efd3ef04f9153d47a990bd7b048a4b2d213daaa5fb8ed670fb85f13bdbcf) // they tried to consolidate 10 50k coins into one 500k coin, and ended up with 50k in change @@ -525,52 +525,52 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) for (int j = 0; j < 20; j++) add_coin(coins, *wallet, 50000 * COIN); - const auto result19 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 500000 * COIN); + const auto result19 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 500000 * COIN, CENT); BOOST_CHECK(result19); BOOST_CHECK_EQUAL(result19->GetSelectedValue(), 500000 * COIN); // we should get the exact amount BOOST_CHECK_EQUAL(result19->GetInputSet().size(), 10U); // in ten coins - // if there's not enough in the smaller coins to make at least 1 * MIN_CHANGE change (0.5+0.6+0.7 < 1.0+1.0), + // if there's not enough in the smaller coins to make at least 1 * CENT change (0.5+0.6+0.7 < 1.0+1.0), // we need to try finding an exact subset anyway // sometimes it will fail, and so we use the next biggest coin: coins.clear(); - add_coin(coins, *wallet, MIN_CHANGE * 5 / 10); - add_coin(coins, *wallet, MIN_CHANGE * 6 / 10); - add_coin(coins, *wallet, MIN_CHANGE * 7 / 10); - add_coin(coins, *wallet, 1111 * MIN_CHANGE); - const auto result20 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 1 * MIN_CHANGE); + add_coin(coins, *wallet, CENT * 5 / 10); + add_coin(coins, *wallet, CENT * 6 / 10); + add_coin(coins, *wallet, CENT * 7 / 10); + add_coin(coins, *wallet, 1111 * CENT); + const auto result20 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 1 * CENT, CENT); BOOST_CHECK(result20); - BOOST_CHECK_EQUAL(result20->GetSelectedValue(), 1111 * MIN_CHANGE); // we get the bigger coin + BOOST_CHECK_EQUAL(result20->GetSelectedValue(), 1111 * CENT); // we get the bigger coin BOOST_CHECK_EQUAL(result20->GetInputSet().size(), 1U); // but sometimes it's possible, and we use an exact subset (0.4 + 0.6 = 1.0) coins.clear(); - add_coin(coins, *wallet, MIN_CHANGE * 4 / 10); - add_coin(coins, *wallet, MIN_CHANGE * 6 / 10); - add_coin(coins, *wallet, MIN_CHANGE * 8 / 10); - add_coin(coins, *wallet, 1111 * MIN_CHANGE); - const auto result21 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), MIN_CHANGE); + add_coin(coins, *wallet, CENT * 4 / 10); + add_coin(coins, *wallet, CENT * 6 / 10); + add_coin(coins, *wallet, CENT * 8 / 10); + add_coin(coins, *wallet, 1111 * CENT); + const auto result21 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), CENT, CENT); BOOST_CHECK(result21); - BOOST_CHECK_EQUAL(result21->GetSelectedValue(), MIN_CHANGE); // we should get the exact amount + BOOST_CHECK_EQUAL(result21->GetSelectedValue(), CENT); // we should get the exact amount BOOST_CHECK_EQUAL(result21->GetInputSet().size(), 2U); // in two coins 0.4+0.6 // test avoiding small change coins.clear(); - add_coin(coins, *wallet, MIN_CHANGE * 5 / 100); - add_coin(coins, *wallet, MIN_CHANGE * 1); - add_coin(coins, *wallet, MIN_CHANGE * 100); + add_coin(coins, *wallet, CENT * 5 / 100); + add_coin(coins, *wallet, CENT * 1); + add_coin(coins, *wallet, CENT * 100); // trying to make 100.01 from these three coins - const auto result22 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), MIN_CHANGE * 10001 / 100); + const auto result22 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), CENT * 10001 / 100, CENT); BOOST_CHECK(result22); - BOOST_CHECK_EQUAL(result22->GetSelectedValue(), MIN_CHANGE * 10105 / 100); // we should get all coins + BOOST_CHECK_EQUAL(result22->GetSelectedValue(), CENT * 10105 / 100); // we should get all coins BOOST_CHECK_EQUAL(result22->GetInputSet().size(), 3U); // but if we try to make 99.9, we should take the bigger of the two small coins to avoid small change - const auto result23 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), MIN_CHANGE * 9990 / 100); + const auto result23 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), CENT * 9990 / 100, CENT); BOOST_CHECK(result23); - BOOST_CHECK_EQUAL(result23->GetSelectedValue(), 101 * MIN_CHANGE); + BOOST_CHECK_EQUAL(result23->GetSelectedValue(), 101 * CENT); BOOST_CHECK_EQUAL(result23->GetInputSet().size(), 2U); } @@ -583,12 +583,12 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // We only create the wallet once to save time, but we still run the coin selection RUN_TESTS times. for (int i = 0; i < RUN_TESTS; i++) { - const auto result24 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 2000); + const auto result24 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 2000, CENT); BOOST_CHECK(result24); - if (amt - 2000 < MIN_CHANGE) { + if (amt - 2000 < CENT) { // needs more than one input: - uint16_t returnSize = std::ceil((2000.0 + MIN_CHANGE)/amt); + uint16_t returnSize = std::ceil((2000.0 + CENT)/amt); CAmount returnValue = amt * returnSize; BOOST_CHECK_EQUAL(result24->GetSelectedValue(), returnValue); BOOST_CHECK_EQUAL(result24->GetInputSet().size(), returnSize); @@ -610,9 +610,9 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) for (int i = 0; i < RUN_TESTS; i++) { // picking 50 from 100 coins doesn't depend on the shuffle, // but does depend on randomness in the stochastic approximation code - const auto result25 = KnapsackSolver(GroupCoins(coins), 50 * COIN); + const auto result25 = KnapsackSolver(GroupCoins(coins), 50 * COIN, CENT); BOOST_CHECK(result25); - const auto result26 = KnapsackSolver(GroupCoins(coins), 50 * COIN); + const auto result26 = KnapsackSolver(GroupCoins(coins), 50 * COIN, CENT); BOOST_CHECK(result26); BOOST_CHECK(!EqualResult(*result25, *result26)); @@ -623,9 +623,9 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // When choosing 1 from 100 identical coins, 1% of the time, this test will choose the same coin twice // which will cause it to fail. // To avoid that issue, run the test RANDOM_REPEATS times and only complain if all of them fail - const auto result27 = KnapsackSolver(GroupCoins(coins), COIN); + const auto result27 = KnapsackSolver(GroupCoins(coins), COIN, CENT); BOOST_CHECK(result27); - const auto result28 = KnapsackSolver(GroupCoins(coins), COIN); + const auto result28 = KnapsackSolver(GroupCoins(coins), COIN, CENT); BOOST_CHECK(result28); if (EqualResult(*result27, *result28)) fails++; @@ -646,9 +646,9 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) int fails = 0; for (int j = 0; j < RANDOM_REPEATS; j++) { - const auto result29 = KnapsackSolver(GroupCoins(coins), 90 * CENT); + const auto result29 = KnapsackSolver(GroupCoins(coins), 90 * CENT, CENT); BOOST_CHECK(result29); - const auto result30 = KnapsackSolver(GroupCoins(coins), 90 * CENT); + const auto result30 = KnapsackSolver(GroupCoins(coins), 90 * CENT, CENT); BOOST_CHECK(result30); if (EqualResult(*result29, *result30)) fails++; @@ -660,6 +660,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) BOOST_AUTO_TEST_CASE(ApproximateBestSubset) { + FastRandomContext rand{}; std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()); wallet->LoadWallet(); LOCK(wallet->cs_wallet); @@ -673,7 +674,7 @@ BOOST_AUTO_TEST_CASE(ApproximateBestSubset) add_coin(coins, *wallet, 1000 * COIN); add_coin(coins, *wallet, 3 * COIN); - const auto result = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 1003 * COIN); + const auto result = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 1003 * COIN, CENT, rand); BOOST_CHECK(result); BOOST_CHECK_EQUAL(result->GetSelectedValue(), 1003 * COIN); BOOST_CHECK_EQUAL(result->GetInputSet().size(), 2U); @@ -714,10 +715,17 @@ BOOST_AUTO_TEST_CASE(SelectCoins_test) CAmount target = rand.randrange(balance - 1000) + 1000; // Perform selection - CoinSelectionParams cs_params(/* change_output_size= */ 34, - /* change_spend_size= */ 148, /* effective_feerate= */ CFeeRate(0), - /* long_term_feerate= */ CFeeRate(0), /* discard_feerate= */ CFeeRate(0), - /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); + CoinSelectionParams cs_params{ + rand, + /*change_output_size=*/ 34, + /*change_spend_size=*/ 148, + /*min_change_target=*/ CENT, + /*effective_feerate=*/ CFeeRate(0), + /*long_term_feerate=*/ CFeeRate(0), + /*discard_feerate=*/ CFeeRate(0), + /*tx_noinputs_size=*/ 0, + /*avoid_partial=*/ false, + }; CCoinControl cc; const auto result = SelectCoins(*wallet, coins, target, cc, cs_params); BOOST_CHECK(result); diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index 35ae3707f8..f61808c549 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -15,22 +15,22 @@ namespace wallet { BOOST_FIXTURE_TEST_SUITE(db_tests, BasicTestingSetup) -static std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& path, std::string& database_filename) +static std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& path, fs::path& database_filename) { fs::path data_file = BDBDataFile(path); - database_filename = fs::PathToString(data_file.filename()); - return GetBerkeleyEnv(data_file.parent_path()); + database_filename = data_file.filename(); + return GetBerkeleyEnv(data_file.parent_path(), false); } BOOST_AUTO_TEST_CASE(getwalletenv_file) { - std::string test_name = "test_name.dat"; - const fs::path datadir = gArgs.GetDataDirNet(); + fs::path test_name = "test_name.dat"; + const fs::path datadir = m_args.GetDataDirNet(); fs::path file_path = datadir / test_name; std::ofstream f{file_path}; f.close(); - std::string filename; + fs::path filename; std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, filename); BOOST_CHECK_EQUAL(filename, test_name); BOOST_CHECK_EQUAL(env->Directory(), datadir); @@ -38,10 +38,10 @@ BOOST_AUTO_TEST_CASE(getwalletenv_file) BOOST_AUTO_TEST_CASE(getwalletenv_directory) { - std::string expected_name = "wallet.dat"; - const fs::path datadir = gArgs.GetDataDirNet(); + fs::path expected_name = "wallet.dat"; + const fs::path datadir = m_args.GetDataDirNet(); - std::string filename; + fs::path filename; std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(datadir, filename); BOOST_CHECK_EQUAL(filename, expected_name); BOOST_CHECK_EQUAL(env->Directory(), datadir); @@ -49,9 +49,9 @@ BOOST_AUTO_TEST_CASE(getwalletenv_directory) BOOST_AUTO_TEST_CASE(getwalletenv_g_dbenvs_multiple) { - fs::path datadir = gArgs.GetDataDirNet() / "1"; - fs::path datadir_2 = gArgs.GetDataDirNet() / "2"; - std::string filename; + fs::path datadir = m_args.GetDataDirNet() / "1"; + fs::path datadir_2 = m_args.GetDataDirNet() / "2"; + fs::path filename; std::shared_ptr<BerkeleyEnvironment> env_1 = GetWalletEnv(datadir, filename); std::shared_ptr<BerkeleyEnvironment> env_2 = GetWalletEnv(datadir, filename); @@ -65,7 +65,7 @@ BOOST_AUTO_TEST_CASE(getwalletenv_g_dbenvs_free_instance) { fs::path datadir = gArgs.GetDataDirNet() / "1"; fs::path datadir_2 = gArgs.GetDataDirNet() / "2"; - std::string filename; + fs::path filename; std::shared_ptr <BerkeleyEnvironment> env_1_a = GetWalletEnv(datadir, filename); std::shared_ptr <BerkeleyEnvironment> env_2_a = GetWalletEnv(datadir_2, filename); diff --git a/src/wallet/test/fuzz/coinselection.cpp b/src/wallet/test/fuzz/coinselection.cpp new file mode 100644 index 0000000000..2693b68cca --- /dev/null +++ b/src/wallet/test/fuzz/coinselection.cpp @@ -0,0 +1,99 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <policy/feerate.h> +#include <primitives/transaction.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/setup_common.h> +#include <wallet/coinselection.h> + +#include <vector> + +namespace wallet { + +static void AddCoin(const CAmount& value, int n_input, int n_input_bytes, int locktime, std::vector<COutput>& coins) +{ + CMutableTransaction tx; + tx.vout.resize(n_input + 1); + tx.vout[n_input].nValue = value; + tx.nLockTime = locktime; // all transactions get different hashes + coins.emplace_back(COutPoint(tx.GetHash(), n_input), tx.vout.at(n_input), /*depth=*/0, n_input_bytes, /*spendable=*/true, /*solvable=*/true, /*safe=*/true, /*time=*/0, /*from_me=*/true); +} + +// Randomly distribute coins to instances of OutputGroup +static void GroupCoins(FuzzedDataProvider& fuzzed_data_provider, const std::vector<COutput>& coins, const CoinSelectionParams& coin_params, bool positive_only, std::vector<OutputGroup>& output_groups) +{ + auto output_group = OutputGroup(coin_params); + bool valid_outputgroup{false}; + for (auto& coin : coins) { + output_group.Insert(coin, /*ancestors=*/0, /*descendants=*/0, positive_only); + // If positive_only was specified, nothing may have been inserted, leading to an empty outpout group + // that would be invalid for the BnB algorithm + valid_outputgroup = !positive_only || output_group.GetSelectionAmount() > 0; + if (valid_outputgroup && fuzzed_data_provider.ConsumeBool()) { + output_groups.push_back(output_group); + output_group = OutputGroup(coin_params); + valid_outputgroup = false; + } + } + if (valid_outputgroup) output_groups.push_back(output_group); +} + +FUZZ_TARGET(coinselection) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + std::vector<COutput> utxo_pool; + + const CFeeRate long_term_fee_rate{ConsumeMoney(fuzzed_data_provider, /*max=*/COIN)}; + const CFeeRate effective_fee_rate{ConsumeMoney(fuzzed_data_provider, /*max=*/COIN)}; + const CAmount cost_of_change{ConsumeMoney(fuzzed_data_provider, /*max=*/COIN)}; + const CAmount target{fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(1, MAX_MONEY)}; + const bool subtract_fee_outputs{fuzzed_data_provider.ConsumeBool()}; + + FastRandomContext fast_random_context{ConsumeUInt256(fuzzed_data_provider)}; + CoinSelectionParams coin_params{fast_random_context}; + coin_params.m_subtract_fee_outputs = subtract_fee_outputs; + coin_params.m_long_term_feerate = long_term_fee_rate; + coin_params.m_effective_feerate = effective_fee_rate; + + // Create some coins + CAmount total_balance{0}; + int next_locktime{0}; + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) + { + const int n_input{fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 10)}; + const int n_input_bytes{fuzzed_data_provider.ConsumeIntegralInRange<int>(100, 10000)}; + const CAmount amount{fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(1, MAX_MONEY)}; + if (total_balance + amount >= MAX_MONEY) { + break; + } + AddCoin(amount, n_input, n_input_bytes, ++next_locktime, utxo_pool); + total_balance += amount; + } + + std::vector<OutputGroup> group_pos; + GroupCoins(fuzzed_data_provider, utxo_pool, coin_params, /*positive_only=*/true, group_pos); + std::vector<OutputGroup> group_all; + GroupCoins(fuzzed_data_provider, utxo_pool, coin_params, /*positive_only=*/false, group_all); + + // Run coinselection algorithms + const auto result_bnb = SelectCoinsBnB(group_pos, target, cost_of_change); + + auto result_srd = SelectCoinsSRD(group_pos, target, fast_random_context); + if (result_srd) result_srd->ComputeAndSetWaste(cost_of_change); + + CAmount change_target{GenerateChangeTarget(target, fast_random_context)}; + auto result_knapsack = KnapsackSolver(group_all, target, change_target, fast_random_context); + if (result_knapsack) result_knapsack->ComputeAndSetWaste(cost_of_change); + + // If the total balance is sufficient for the target and we are not using + // effective values, Knapsack should always find a solution. + if (total_balance >= target && subtract_fee_outputs) { + assert(result_knapsack); + } +} + +} // namespace wallet diff --git a/src/wallet/test/fuzz/notifications.cpp b/src/wallet/test/fuzz/notifications.cpp index 1c16da25bd..9089c8ff46 100644 --- a/src/wallet/test/fuzz/notifications.cpp +++ b/src/wallet/test/fuzz/notifications.cpp @@ -64,7 +64,7 @@ struct FuzzedWallet { assert(RemoveWallet(context, wallet, load_on_start, warnings)); assert(warnings.empty()); UnloadWallet(std::move(wallet)); - fs::remove_all(GetWalletDir() / name); + fs::remove_all(GetWalletDir() / fs::PathFromString(name)); } CScript GetScriptPubKey(FuzzedDataProvider& fuzzed_data_provider) { diff --git a/src/wallet/test/init_test_fixture.cpp b/src/wallet/test/init_test_fixture.cpp index be38cebafd..8eb7689c94 100644 --- a/src/wallet/test/init_test_fixture.cpp +++ b/src/wallet/test/init_test_fixture.cpp @@ -15,20 +15,19 @@ namespace wallet { InitWalletDirTestingSetup::InitWalletDirTestingSetup(const std::string& chainName) : BasicTestingSetup(chainName) { - m_wallet_loader = MakeWalletLoader(*m_node.chain, *Assert(m_node.args)); + m_wallet_loader = MakeWalletLoader(*m_node.chain, m_args); - std::string sep; - sep += fs::path::preferred_separator; + const auto sep = fs::path::preferred_separator; - m_datadir = gArgs.GetDataDirNet(); + m_datadir = m_args.GetDataDirNet(); m_cwd = fs::current_path(); m_walletdir_path_cases["default"] = m_datadir / "wallets"; m_walletdir_path_cases["custom"] = m_datadir / "my_wallets"; m_walletdir_path_cases["nonexistent"] = m_datadir / "path_does_not_exist"; m_walletdir_path_cases["file"] = m_datadir / "not_a_directory.dat"; - m_walletdir_path_cases["trailing"] = m_datadir / ("wallets" + sep); - m_walletdir_path_cases["trailing2"] = m_datadir / ("wallets" + sep + sep); + m_walletdir_path_cases["trailing"] = (m_datadir / "wallets") + sep; + m_walletdir_path_cases["trailing2"] = (m_datadir / "wallets") + sep + sep; fs::current_path(m_datadir); m_walletdir_path_cases["relative"] = "wallets"; @@ -42,14 +41,11 @@ InitWalletDirTestingSetup::InitWalletDirTestingSetup(const std::string& chainNam InitWalletDirTestingSetup::~InitWalletDirTestingSetup() { - gArgs.LockSettings([&](util::Settings& settings) { - settings.forced_settings.erase("walletdir"); - }); fs::current_path(m_cwd); } void InitWalletDirTestingSetup::SetWalletDir(const fs::path& walletdir_path) { - gArgs.ForceSetArg("-walletdir", fs::PathToString(walletdir_path)); + m_args.ForceSetArg("-walletdir", fs::PathToString(walletdir_path)); } } // namespace wallet diff --git a/src/wallet/test/init_tests.cpp b/src/wallet/test/init_tests.cpp index 7fdecc5642..fb0a07e181 100644 --- a/src/wallet/test/init_tests.cpp +++ b/src/wallet/test/init_tests.cpp @@ -18,7 +18,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_default) SetWalletDir(m_walletdir_path_cases["default"]); bool result = m_wallet_loader->verify(); BOOST_CHECK(result == true); - fs::path walletdir = gArgs.GetPathArg("-walletdir"); + fs::path walletdir = m_args.GetPathArg("-walletdir"); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); BOOST_CHECK_EQUAL(walletdir, expected_path); } @@ -28,7 +28,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_custom) SetWalletDir(m_walletdir_path_cases["custom"]); bool result = m_wallet_loader->verify(); BOOST_CHECK(result == true); - fs::path walletdir = gArgs.GetPathArg("-walletdir"); + fs::path walletdir = m_args.GetPathArg("-walletdir"); fs::path expected_path = fs::canonical(m_walletdir_path_cases["custom"]); BOOST_CHECK_EQUAL(walletdir, expected_path); } @@ -68,7 +68,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing) SetWalletDir(m_walletdir_path_cases["trailing"]); bool result = m_wallet_loader->verify(); BOOST_CHECK(result == true); - fs::path walletdir = gArgs.GetPathArg("-walletdir"); + fs::path walletdir = m_args.GetPathArg("-walletdir"); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); BOOST_CHECK_EQUAL(walletdir, expected_path); } @@ -78,7 +78,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing2) SetWalletDir(m_walletdir_path_cases["trailing2"]); bool result = m_wallet_loader->verify(); BOOST_CHECK(result == true); - fs::path walletdir = gArgs.GetPathArg("-walletdir"); + fs::path walletdir = m_args.GetPathArg("-walletdir"); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); BOOST_CHECK_EQUAL(walletdir, expected_path); } diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index b953f402a2..62053ae8d2 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -68,7 +68,6 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) // Try to sign the mutated input SignatureData sigdata; BOOST_CHECK(m_wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, true, true) != TransactionError::OK); - //BOOST_CHECK(spk_man->FillPSBT(psbtx, PrecomputePSBTData(psbtx), SIGHASH_ALL, true, true) != TransactionError::OK); } BOOST_AUTO_TEST_CASE(parse_hd_keypath) diff --git a/src/wallet/test/wallet_crypto_tests.cpp b/src/wallet/test/wallet_crypto_tests.cpp index 166e27bab9..327c28412a 100644 --- a/src/wallet/test/wallet_crypto_tests.cpp +++ b/src/wallet/test/wallet_crypto_tests.cpp @@ -81,7 +81,7 @@ BOOST_AUTO_TEST_CASE(passphrase) { std::string hash(GetRandHash().ToString()); std::vector<unsigned char> vchSalt(8); - GetRandBytes(vchSalt.data(), vchSalt.size()); + GetRandBytes(vchSalt); uint32_t rounds = InsecureRand32(); if (rounds > 30000) rounds = 30000; diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index cb006dea3a..38d23b6f91 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -9,6 +9,7 @@ namespace wallet { WalletTestingSetup::WalletTestingSetup(const std::string& chainName) : TestingSetup(chainName), + m_wallet_loader{interfaces::MakeWalletLoader(*m_node.chain, *Assert(m_node.args))}, m_wallet(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()) { m_wallet.LoadWallet(); diff --git a/src/wallet/test/wallet_test_fixture.h b/src/wallet/test/wallet_test_fixture.h index d4b855b145..e0b38a40e7 100644 --- a/src/wallet/test/wallet_test_fixture.h +++ b/src/wallet/test/wallet_test_fixture.h @@ -22,7 +22,7 @@ struct WalletTestingSetup : public TestingSetup { explicit WalletTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); ~WalletTestingSetup(); - std::unique_ptr<interfaces::WalletLoader> m_wallet_loader = interfaces::MakeWalletLoader(*m_node.chain, *Assert(m_node.args)); + std::unique_ptr<interfaces::WalletLoader> m_wallet_loader; CWallet m_wallet; std::unique_ptr<interfaces::Handler> m_chain_notifications_handler; }; diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 7693c9c0e8..683f0eb327 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -54,6 +54,7 @@ static const std::shared_ptr<CWallet> TestLoadWallet(WalletContext& context) std::vector<bilingual_str> warnings; auto database = MakeWalletDatabase("", options, status, error); auto wallet = CWallet::Create(context, "", std::move(database), options.create_flags, error, warnings); + NotifyWalletLoaded(context, wallet); if (context.chain) { wallet->postInitProcess(); } @@ -221,7 +222,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) wallet->SetupLegacyScriptPubKeyMan(); WITH_LOCK(wallet->cs_wallet, wallet->SetLastBlockProcessed(newTip->nHeight, newTip->GetBlockHash())); WalletContext context; - context.args = &gArgs; + context.args = &m_args; AddWallet(context, wallet); UniValue keys; keys.setArray(); @@ -277,12 +278,12 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) SetMockTime(KEY_TIME); m_coinbase_txns.emplace_back(CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); - std::string backup_file = fs::PathToString(gArgs.GetDataDirNet() / "wallet.backup"); + std::string backup_file = fs::PathToString(m_args.GetDataDirNet() / "wallet.backup"); // Import key into wallet and call dumpwallet to create backup file. { WalletContext context; - context.args = &gArgs; + context.args = &m_args; const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); { auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); @@ -310,7 +311,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) wallet->SetupLegacyScriptPubKeyMan(); WalletContext context; - context.args = &gArgs; + context.args = &m_args; JSONRPCRequest request; request.context = &context; request.params.setArray(); @@ -339,7 +340,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) { CWallet wallet(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); - CWalletTx wtx{m_coinbase_txns.back(), TxStateConfirmed{m_node.chainman->ActiveChain().Tip()->GetBlockHash(), m_node.chainman->ActiveChain().Height(), /*position_in_block=*/0}}; + CWalletTx wtx{m_coinbase_txns.back(), TxStateConfirmed{m_node.chainman->ActiveChain().Tip()->GetBlockHash(), m_node.chainman->ActiveChain().Height(), /*index=*/0}}; LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -367,13 +368,13 @@ static int64_t AddTx(ChainstateManager& chainman, CWallet& wallet, uint32_t lock CBlockIndex* block = nullptr; if (blockTime > 0) { LOCK(cs_main); - auto inserted = chainman.BlockIndex().emplace(GetRandHash(), new CBlockIndex); + auto inserted = chainman.BlockIndex().emplace(std::piecewise_construct, std::make_tuple(GetRandHash()), std::make_tuple()); assert(inserted.second); const uint256& hash = inserted.first->first; - block = inserted.first->second; + block = &inserted.first->second; block->nTime = blockTime; block->phashBlock = &hash; - state = TxStateConfirmed{hash, block->nHeight, /*position_in_block=*/0}; + state = TxStateConfirmed{hash, block->nHeight, /*index=*/0}; } return wallet.AddToWallet(MakeTransactionRef(tx), state, [&](CWalletTx& wtx, bool /* new_tx */) { // Assign wtx.m_state to simplify test and avoid the need to simulate @@ -540,7 +541,7 @@ public: wallet->SetLastBlockProcessed(wallet->GetLastBlockHeight() + 1, m_node.chainman->ActiveChain().Tip()->GetBlockHash()); auto it = wallet->mapWallet.find(tx->GetHash()); BOOST_CHECK(it != wallet->mapWallet.end()); - it->second.m_state = TxStateConfirmed{m_node.chainman->ActiveChain().Tip()->GetBlockHash(), m_node.chainman->ActiveChain().Height(), /*position_in_block=*/1}; + it->second.m_state = TxStateConfirmed{m_node.chainman->ActiveChain().Tip()->GetBlockHash(), m_node.chainman->ActiveChain().Height(), /*index=*/1}; return it->second; } @@ -588,7 +589,7 @@ BOOST_FIXTURE_TEST_CASE(ListCoinsTest, ListCoinsTestingSetup) for (const auto& group : list) { for (const auto& coin : group.second) { LOCK(wallet->cs_wallet); - wallet->LockCoin(COutPoint(coin.tx->GetHash(), coin.i)); + wallet->LockCoin(coin.outpoint); } } { @@ -716,10 +717,10 @@ BOOST_FIXTURE_TEST_CASE(wallet_descriptor_test, BasicTestingSetup) //! rescanning where new transactions in new blocks could be lost. BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) { - gArgs.ForceSetArg("-unsafesqlitesync", "1"); + m_args.ForceSetArg("-unsafesqlitesync", "1"); // Create new wallet with known key and unload it. WalletContext context; - context.args = &gArgs; + context.args = &m_args; context.chain = m_node.chain.get(); auto wallet = TestLoadWallet(context); CKey key; @@ -812,7 +813,7 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) BOOST_FIXTURE_TEST_CASE(CreateWalletWithoutChain, BasicTestingSetup) { WalletContext context; - context.args = &gArgs; + context.args = &m_args; auto wallet = TestLoadWallet(context); BOOST_CHECK(wallet); UnloadWallet(std::move(wallet)); @@ -820,9 +821,9 @@ BOOST_FIXTURE_TEST_CASE(CreateWalletWithoutChain, BasicTestingSetup) BOOST_FIXTURE_TEST_CASE(ZapSelectTx, TestChain100Setup) { - gArgs.ForceSetArg("-unsafesqlitesync", "1"); + m_args.ForceSetArg("-unsafesqlitesync", "1"); WalletContext context; - context.args = &gArgs; + context.args = &m_args; context.chain = m_node.chain.get(); auto wallet = TestLoadWallet(context); CKey key; diff --git a/src/wallet/transaction.h b/src/wallet/transaction.h index 00f9c9f154..271d698e56 100644 --- a/src/wallet/transaction.h +++ b/src/wallet/transaction.h @@ -296,6 +296,7 @@ public: bool isUnconfirmed() const { return !isAbandoned() && !isConflicted() && !isConfirmed(); } bool isConfirmed() const { return state<TxStateConfirmed>(); } const uint256& GetHash() const { return tx->GetHash(); } + const uint256& GetWitnessHash() const { return tx->GetWitnessHash(); } bool IsCoinBase() const { return tx->IsCoinBase(); } // Disable copying of CWalletTx objects to prevent bugs where instances get diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 6cf9f9ce74..5a23f739d3 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -44,8 +44,6 @@ #include <assert.h> #include <optional> -#include <boost/algorithm/string/replace.hpp> - using interfaces::FoundBlock; namespace wallet { @@ -167,6 +165,14 @@ std::unique_ptr<interfaces::Handler> HandleLoadWallet(WalletContext& context, Lo return interfaces::MakeHandler([&context, it] { LOCK(context.wallets_mutex); context.wallet_load_fns.erase(it); }); } +void NotifyWalletLoaded(WalletContext& context, const std::shared_ptr<CWallet>& wallet) +{ + LOCK(context.wallets_mutex); + for (auto& load_wallet : context.wallet_load_fns) { + load_wallet(interfaces::MakeWallet(context, wallet)); + } +} + static Mutex g_loading_wallet_mutex; static Mutex g_wallet_release_mutex; static std::condition_variable g_wallet_release_cv; @@ -232,6 +238,8 @@ std::shared_ptr<CWallet> LoadWalletInternal(WalletContext& context, const std::s status = DatabaseStatus::FAILED_LOAD; return nullptr; } + + NotifyWalletLoaded(context, wallet); AddWallet(context, wallet); wallet->postInitProcess(); @@ -289,6 +297,13 @@ std::shared_ptr<CWallet> CreateWallet(WalletContext& context, const std::string& return nullptr; } + // Do not allow a passphrase when private keys are disabled + if (!passphrase.empty() && (wallet_creation_flags & WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + error = Untranslated("Passphrase provided but private keys are disabled. A passphrase is only used to encrypt private keys, so cannot be used for wallets with private keys disabled."); + status = DatabaseStatus::FAILED_CREATE; + return nullptr; + } + // Wallet::Verify will check if we're trying to create a wallet with a duplicate name. std::unique_ptr<WalletDatabase> database = MakeWalletDatabase(name, options, status, error); if (!database) { @@ -297,13 +312,6 @@ std::shared_ptr<CWallet> CreateWallet(WalletContext& context, const std::string& return nullptr; } - // Do not allow a passphrase when private keys are disabled - if (!passphrase.empty() && (wallet_creation_flags & WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { - error = Untranslated("Passphrase provided but private keys are disabled. A passphrase is only used to encrypt private keys, so cannot be used for wallets with private keys disabled."); - status = DatabaseStatus::FAILED_CREATE; - return nullptr; - } - // Make the wallet context.chain->initMessage(_("Loading wallet…").translated); const std::shared_ptr<CWallet> wallet = CWallet::Create(context, name, std::move(database), wallet_creation_flags, error, warnings); @@ -348,12 +356,19 @@ std::shared_ptr<CWallet> CreateWallet(WalletContext& context, const std::string& wallet->Lock(); } } + + NotifyWalletLoaded(context, wallet); AddWallet(context, wallet); wallet->postInitProcess(); // Write the wallet settings UpdateWalletSetting(*context.chain, name, load_on_start, warnings); + // Legacy wallets are being deprecated, warn if a newly created wallet is legacy + if (!(wallet_creation_flags & WALLET_FLAG_DESCRIPTORS)) { + warnings.push_back(_("Wallet created successfully. The legacy wallet type is being deprecated and support for creating and opening legacy wallets will be removed in the future.")); + } + status = DatabaseStatus::SUCCESS; return wallet; } @@ -361,6 +376,7 @@ std::shared_ptr<CWallet> CreateWallet(WalletContext& context, const std::string& std::shared_ptr<CWallet> RestoreWallet(WalletContext& context, const fs::path& backup_file, const std::string& wallet_name, std::optional<bool> load_on_start, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings) { DatabaseOptions options; + ReadDatabaseArgs(*context.args, options); options.require_existing = true; if (!fs::exists(backup_file)) { @@ -505,6 +521,11 @@ bool CWallet::ChangeWalletPassphrase(const SecureString& strOldWalletPassphrase, void CWallet::chainStateFlushed(const CBlockLocator& loc) { + // Don't update the best block until the chain is attached so that in case of a shutdown, + // the rescan will be restarted at next startup. + if (m_attaching_chain) { + return; + } WalletBatch batch(GetDatabase()); batch.WriteBestBlock(loc); } @@ -664,12 +685,12 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) CKeyingMaterial _vMasterKey; _vMasterKey.resize(WALLET_CRYPTO_KEY_SIZE); - GetStrongRandBytes(_vMasterKey.data(), WALLET_CRYPTO_KEY_SIZE); + GetStrongRandBytes(_vMasterKey); CMasterKey kMasterKey; kMasterKey.vchSalt.resize(WALLET_CRYPTO_SALT_SIZE); - GetStrongRandBytes(kMasterKey.vchSalt.data(), WALLET_CRYPTO_SALT_SIZE); + GetStrongRandBytes(kMasterKey.vchSalt); CCrypter crypter; int64_t nStartTime = GetTimeMillis(); @@ -995,14 +1016,14 @@ CWalletTx* CWallet::AddToWallet(CTransactionRef tx, const TxState& state, const if (!strCmd.empty()) { - boost::replace_all(strCmd, "%s", hash.GetHex()); + ReplaceAll(strCmd, "%s", hash.GetHex()); if (auto* conf = wtx.state<TxStateConfirmed>()) { - boost::replace_all(strCmd, "%b", conf->confirmed_block_hash.GetHex()); - boost::replace_all(strCmd, "%h", ToString(conf->confirmed_block_height)); + ReplaceAll(strCmd, "%b", conf->confirmed_block_hash.GetHex()); + ReplaceAll(strCmd, "%h", ToString(conf->confirmed_block_height)); } else { - boost::replace_all(strCmd, "%b", "unconfirmed"); - boost::replace_all(strCmd, "%h", "-1"); + ReplaceAll(strCmd, "%b", "unconfirmed"); + ReplaceAll(strCmd, "%h", "-1"); } #ifndef WIN32 // Substituting the wallet name isn't currently supported on windows @@ -1010,7 +1031,7 @@ CWalletTx* CWallet::AddToWallet(CTransactionRef tx, const TxState& state, const // https://github.com/bitcoin/bitcoin/pull/13339#issuecomment-537384875 // A few ways it could be implemented in the future are described in: // https://github.com/bitcoin/bitcoin/pull/13339#issuecomment-461288094 - boost::replace_all(strCmd, "%w", ShellEscape(GetName())); + ReplaceAll(strCmd, "%w", ShellEscape(GetName())); #endif std::thread t(runCommand, strCmd); t.detach(); // thread runs free @@ -1686,8 +1707,10 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r */ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_block, int start_height, std::optional<int> max_height, const WalletRescanReserver& reserver, bool fUpdate) { - int64_t nNow = GetTime(); - int64_t start_time = GetTimeMillis(); + using Clock = std::chrono::steady_clock; + constexpr auto LOG_INTERVAL{60s}; + auto current_time{Clock::now()}; + auto start_time{Clock::now()}; assert(reserver.isReserved()); @@ -1714,8 +1737,8 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc if (block_height % 100 == 0 && progress_end - progress_begin > 0.0) { ShowProgress(strprintf("%s " + _("Rescanning…").translated, GetDisplayName()), std::max(1, std::min(99, (int)(m_scanning_progress * 100)))); } - if (GetTime() >= nNow + 60) { - nNow = GetTime(); + if (Clock::now() >= current_time + LOG_INTERVAL) { + current_time = Clock::now(); WalletLogPrintf("Still rescanning. At block %d. Progress=%f\n", block_height, progress_current); } @@ -1782,7 +1805,8 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc WalletLogPrintf("Rescan interrupted by shutdown request at block %d. Progress=%f\n", block_height, progress_current); result.status = ScanResult::USER_ABORT; } else { - WalletLogPrintf("Rescan completed in %15dms\n", GetTimeMillis() - start_time); + auto duration_milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(Clock::now() - start_time); + WalletLogPrintf("Rescan completed in %15dms\n", duration_milliseconds.count()); } return result; } @@ -2750,7 +2774,7 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri } else if (wallet_creation_flags & WALLET_FLAG_DISABLE_PRIVATE_KEYS) { // Make it impossible to disable private keys after creation error = strprintf(_("Error loading %s: Private keys can only be disabled during creation"), walletFile); - return NULL; + return nullptr; } else if (walletInstance->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { if (spk_man->HavePrivateKeys()) { @@ -2899,13 +2923,6 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri } { - LOCK(context.wallets_mutex); - for (auto& load_wallet : context.wallet_load_fns) { - load_wallet(interfaces::MakeWallet(context, walletInstance)); - } - } - - { LOCK(walletInstance->cs_wallet); walletInstance->SetBroadcastTransactions(args.GetBoolArg("-walletbroadcast", DEFAULT_WALLETBROADCAST)); walletInstance->WalletLogPrintf("setKeyPool.size() = %u\n", walletInstance->GetKeyPoolSize()); @@ -2923,12 +2940,28 @@ bool CWallet::AttachChain(const std::shared_ptr<CWallet>& walletInstance, interf assert(!walletInstance->m_chain || walletInstance->m_chain == &chain); walletInstance->m_chain = &chain; + // Unless allowed, ensure wallet files are not reused across chains: + if (!gArgs.GetBoolArg("-walletcrosschain", DEFAULT_WALLETCROSSCHAIN)) { + WalletBatch batch(walletInstance->GetDatabase()); + CBlockLocator locator; + if (batch.ReadBestBlock(locator) && locator.vHave.size() > 0 && chain.getHeight()) { + // Wallet is assumed to be from another chain, if genesis block in the active + // chain differs from the genesis block known to the wallet. + if (chain.getBlockHash(0) != locator.vHave.back()) { + error = Untranslated("Wallet files should not be reused across chains. Restart bitcoind with -walletcrosschain to override."); + return false; + } + } + } + // Register wallet with validationinterface. It's done before rescan to avoid // missing block connections between end of rescan and validation subscribing. // Because of wallet lock being hold, block connection notifications are going to // be pending on the validation-side until lock release. It's likely to have // block processing duplicata (if rescan block range overlaps with notification one) // but we guarantee at least than wallet state is correct after notifications delivery. + // However, chainStateFlushed notifications are ignored until the rescan is finished + // so that in case of a shutdown event, the rescan will be repeated at the next start. // This is temporary until rescan and notifications delivery are unified under same // interface. walletInstance->m_chain_notifications_handler = walletInstance->chain().handleNotifications(walletInstance); @@ -2957,6 +2990,7 @@ bool CWallet::AttachChain(const std::shared_ptr<CWallet>& walletInstance, interf if (tip_height && *tip_height != rescan_height) { + walletInstance->m_attaching_chain = true; //ignores chainStateFlushed notifications if (chain.havePruned()) { int block_height = *tip_height; while (block_height > 0 && chain.haveBlockOnDisk(block_height - 1) && rescan_height != block_height) { @@ -2996,6 +3030,7 @@ bool CWallet::AttachChain(const std::shared_ptr<CWallet>& walletInstance, interf return false; } } + walletInstance->m_attaching_chain = false; walletInstance->chainStateFlushed(chain.getTipLocator()); walletInstance->GetDatabase().IncrementUpdateCounter(); } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index e2c5c69c91..4e81a2b957 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -20,7 +20,6 @@ #include <util/system.h> #include <util/ui_change_type.h> #include <validationinterface.h> -#include <wallet/coinselection.h> #include <wallet/crypter.h> #include <wallet/scriptpubkeyman.h> #include <wallet/transaction.h> @@ -68,6 +67,7 @@ std::shared_ptr<CWallet> LoadWallet(WalletContext& context, const std::string& n std::shared_ptr<CWallet> CreateWallet(WalletContext& context, const std::string& name, std::optional<bool> load_on_start, DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings); std::shared_ptr<CWallet> RestoreWallet(WalletContext& context, const fs::path& backup_file, const std::string& wallet_name, std::optional<bool> load_on_start, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings); std::unique_ptr<interfaces::Handler> HandleLoadWallet(WalletContext& context, LoadWalletFn load_wallet); +void NotifyWalletLoaded(WalletContext& context, const std::shared_ptr<CWallet>& wallet); std::unique_ptr<WalletDatabase> MakeWalletDatabase(const std::string& name, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); //! -paytxfee default @@ -95,13 +95,14 @@ static const CAmount WALLET_INCREMENTAL_RELAY_FEE = 5000; //! Default for -spendzeroconfchange static const bool DEFAULT_SPEND_ZEROCONF_CHANGE = true; //! Default for -walletrejectlongchains -static const bool DEFAULT_WALLET_REJECT_LONG_CHAINS = false; +static const bool DEFAULT_WALLET_REJECT_LONG_CHAINS{true}; //! -txconfirmtarget default static const unsigned int DEFAULT_TX_CONFIRM_TARGET = 6; //! -walletrbf default static const bool DEFAULT_WALLET_RBF = false; static const bool DEFAULT_WALLETBROADCAST = true; static const bool DEFAULT_DISABLE_WALLET = false; +static const bool DEFAULT_WALLETCROSSCHAIN = false; //! -maxtxfee default constexpr CAmount DEFAULT_TRANSACTION_MAXFEE{COIN / 10}; //! Discourage users to set fees higher than this amount (in satoshis) per kB @@ -112,7 +113,6 @@ constexpr CAmount HIGH_MAX_TX_FEE{100 * HIGH_TX_FEE_PER_KB}; static constexpr size_t DUMMY_NESTED_P2WPKH_INPUT_SIZE = 91; class CCoinControl; -class COutput; class CWalletTx; class ReserveDestination; @@ -238,6 +238,7 @@ private: std::atomic<bool> fAbortRescan{false}; std::atomic<bool> fScanningWallet{false}; // controlled by WalletRescanReserver + std::atomic<bool> m_attaching_chain{false}; std::atomic<int64_t> m_scanning_start{0}; std::atomic<double> m_scanning_progress{0}; friend class WalletRescanReserver; diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 6e100524a4..7bbed7973f 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -850,10 +850,10 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) // Set the active ScriptPubKeyMans for (auto spk_man_pair : wss.m_active_external_spks) { - pwallet->LoadActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /* internal */ false); + pwallet->LoadActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /*internal=*/false); } for (auto spk_man_pair : wss.m_active_internal_spks) { - pwallet->LoadActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /* internal */ true); + pwallet->LoadActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /*internal=*/true); } // Set the descriptor caches @@ -1189,10 +1189,11 @@ std::unique_ptr<WalletDatabase> CreateDummyWalletDatabase() /** Return object for accessing temporary in-memory database. */ std::unique_ptr<WalletDatabase> CreateMockWalletDatabase() { + DatabaseOptions options; #ifdef USE_SQLITE - return std::make_unique<SQLiteDatabase>("", "", true); + return std::make_unique<SQLiteDatabase>("", "", options, true); #elif USE_BDB - return std::make_unique<BerkeleyDatabase>(std::make_shared<BerkeleyEnvironment>(), ""); + return std::make_unique<BerkeleyDatabase>(std::make_shared<BerkeleyEnvironment>(), "", options); #endif } } // namespace wallet diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 9cd18dd0a5..769175b5a8 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -140,6 +140,7 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) if (command == "create") { DatabaseOptions options; + ReadDatabaseArgs(args, options); options.require_create = true; // If -legacy is set, use it. Otherwise default to false. bool make_legacy = args.GetBoolArg("-legacy", false); @@ -165,6 +166,7 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) } } else if (command == "info") { DatabaseOptions options; + ReadDatabaseArgs(args, options); options.require_existing = true; const std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, args, options); if (!wallet_instance) return false; @@ -174,7 +176,7 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) #ifdef USE_BDB bilingual_str error; std::vector<bilingual_str> warnings; - bool ret = RecoverDatabaseFile(path, error, warnings); + bool ret = RecoverDatabaseFile(args, path, error, warnings); if (!ret) { for (const auto& warning : warnings) { tfm::format(std::cerr, "%s\n", warning.original); @@ -190,11 +192,12 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) #endif } else if (command == "dump") { DatabaseOptions options; + ReadDatabaseArgs(args, options); options.require_existing = true; const std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, args, options); if (!wallet_instance) return false; bilingual_str error; - bool ret = DumpWallet(*wallet_instance, error); + bool ret = DumpWallet(args, *wallet_instance, error); if (!ret && !error.empty()) { tfm::format(std::cerr, "%s\n", error.original); return ret; @@ -204,7 +207,7 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) } else if (command == "createfromdump") { bilingual_str error; std::vector<bilingual_str> warnings; - bool ret = CreateFromDump(name, path, error, warnings); + bool ret = CreateFromDump(args, name, path, error, warnings); for (const auto& warning : warnings) { tfm::format(std::cout, "%s\n", warning.original); } diff --git a/src/zmq/zmqrpc.cpp b/src/zmq/zmqrpc.cpp index f9f8b5a9dc..ec6d1cbba3 100644 --- a/src/zmq/zmqrpc.cpp +++ b/src/zmq/zmqrpc.cpp @@ -51,10 +51,8 @@ static RPCHelpMan getzmqnotifications() }; } -const CRPCCommand commands[] = -{ // category actor (function) - // ----------------- ----------------------- - { "zmq", &getzmqnotifications, }, +const CRPCCommand commands[]{ + {"zmq", &getzmqnotifications}, }; } // anonymous namespace diff --git a/test/README.md b/test/README.md index 8fffde888d..0d9b9fb89b 100644 --- a/test/README.md +++ b/test/README.md @@ -98,7 +98,7 @@ test/functional/test_runner.py --extended In order to run backwards compatibility tests, download the previous node binaries: ``` -test/get_previous_releases.py -b v22.0 v0.21.0 v0.20.1 v0.19.1 v0.18.1 v0.17.2 v0.16.3 v0.15.2 +test/get_previous_releases.py -b v23.0 v22.0 v0.21.0 v0.20.1 v0.19.1 v0.18.1 v0.17.2 v0.16.3 v0.15.2 v0.14.3 ``` By default, up to 4 tests will be run in parallel by test_runner. To specify @@ -107,6 +107,34 @@ how many jobs to run, append `--jobs=n` The individual tests and the test_runner harness have many command-line options. Run `test/functional/test_runner.py -h` to see them all. +#### Speed up test runs with a ramdisk + +If you have available RAM on your system you can create a ramdisk to use as the `cache` and `tmp` directories for the functional tests in order to speed them up. +Speed-up amount varies on each system (and according to your ram speed and other variables), but a 2-3x speed-up is not uncommon. + +To create a 4GB ramdisk on Linux at `/mnt/tmp/`: + +```bash +sudo mkdir -p /mnt/tmp +sudo mount -t tmpfs -o size=4g tmpfs /mnt/tmp/ +``` + +Configure the size of the ramdisk using the `size=` option. +The size of the ramdisk needed is relative to the number of concurrent jobs the test suite runs. +For example running the test suite with `--jobs=100` might need a 4GB ramdisk, but running with `--jobs=32` will only need a 2.5GB ramdisk. + +To use, run the test suite specifying the ramdisk as the `cachedir` and `tmpdir`: + +```bash +test/functional/test_runner.py --cachedir=/mnt/tmp/cache --tmpdir=/mnt/tmp +``` + +Once finished with the tests and the disk, and to free the ram, simply unmount the disk: + +```bash +sudo umount /mnt/tmp +``` + #### Troubleshooting and debugging test failures ##### Resource contention @@ -277,11 +305,12 @@ Use the `-v` option for verbose output. | Lint test | Dependency | |-----------|:----------:| -| [`lint-python.sh`](lint/lint-python.sh) | [flake8](https://gitlab.com/pycqa/flake8) -| [`lint-python.sh`](lint/lint-python.sh) | [mypy](https://github.com/python/mypy) -| [`lint-python.sh`](lint/lint-python.sh) | [pyzmq](https://github.com/zeromq/pyzmq) -| [`lint-shell.sh`](lint/lint-shell.sh) | [ShellCheck](https://github.com/koalaman/shellcheck) -| [`lint-spelling.sh`](lint/lint-spelling.sh) | [codespell](https://github.com/codespell-project/codespell) +| [`lint-python.py`](lint/lint-python.py) | [flake8](https://gitlab.com/pycqa/flake8) +| [`lint-python.py`](lint/lint-python.py) | [mypy](https://github.com/python/mypy) +| [`lint-python.py`](lint/lint-python.py) | [pyzmq](https://github.com/zeromq/pyzmq) +| [`lint-python-dead-code.py`](lint/lint-python-dead-code.py) | [vulture](https://github.com/jendrikseipp/vulture) +| [`lint-shell.py`](lint/lint-shell.py) | [ShellCheck](https://github.com/koalaman/shellcheck) +| [`lint-spelling.py`](lint/lint-spelling.py) | [codespell](https://github.com/codespell-project/codespell) In use versions and install instructions are available in the [CI setup](../ci/lint/04_install.sh). @@ -292,13 +321,13 @@ Please be aware that on Linux distributions all dependencies are usually availab Individual tests can be run by directly calling the test script, e.g.: ``` -test/lint/lint-files.sh +test/lint/lint-files.py ``` You can run all the shell-based lint tests by running: ``` -test/lint/lint-all.sh +test/lint/lint-all.py ``` # Writing functional tests diff --git a/test/config.ini.in b/test/config.ini.in index 8bcba1b39c..5888ef443b 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -19,9 +19,11 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py @USE_SQLITE_TRUE@USE_SQLITE=true @USE_BDB_TRUE@USE_BDB=true @BUILD_BITCOIN_CLI_TRUE@ENABLE_CLI=true +@BUILD_BITCOIN_UTIL_TRUE@ENABLE_BITCOIN_UTIL=true @BUILD_BITCOIN_WALLET_TRUE@ENABLE_WALLET_TOOL=true @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true @ENABLE_FUZZ_TRUE@ENABLE_FUZZ=true @ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true @ENABLE_EXTERNAL_SIGNER_TRUE@ENABLE_EXTERNAL_SIGNER=true @ENABLE_SYSCALL_SANDBOX_TRUE@ENABLE_SYSCALL_SANDBOX=true +@ENABLE_USDT_TRACEPOINTS_TRUE@ENABLE_USDT_TRACEPOINTS=true diff --git a/test/functional/README.md b/test/functional/README.md index 926810cf03..914dbfd977 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -24,7 +24,7 @@ don't have test cases for. Consider using [pyenv](https://github.com/pyenv/pyenv), which checks [.python-version](/.python-version), to prevent accidentally introducing modern syntax from an unsupported Python version. The CI linter job also checks this, but [possibly not in all cases](https://github.com/bitcoin/bitcoin/pull/14884#discussion_r239585126). -- See [the python lint script](/test/lint/lint-python.sh) that checks for violations that +- See [the python lint script](/test/lint/lint-python.py) that checks for violations that could lead to bugs and issues in the test code. - Use [type hints](https://docs.python.org/3/library/typing.html) in your code to improve code readability and to detect possible bugs earlier. diff --git a/test/functional/feature_addrman.py b/test/functional/feature_addrman.py index 0fdefaa9c3..5e49d0214a 100755 --- a/test/functional/feature_addrman.py +++ b/test/functional/feature_addrman.py @@ -68,6 +68,17 @@ class AddrmanTest(BitcoinTestFramework): self.start_node(0, extra_args=["-checkaddrman=1"]) assert_equal(self.nodes[0].getnodeaddresses(), []) + self.log.info("Check that addrman with negative lowest_compatible cannot be read") + self.stop_node(0) + write_addrman(peers_dat, lowest_compatible=-32) + self.nodes[0].assert_start_raises_init_error( + expected_msg=init_error( + "Corrupted addrman database: The compat value \\(0\\) is lower " + "than the expected minimum value 32.: (.+)" + ), + match=ErrorMatch.FULL_REGEX, + ) + self.log.info("Check that addrman from future is overwritten with new addrman") self.stop_node(0) write_addrman(peers_dat, lowest_compatible=111) diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py index a7fb3184a6..59a12193fd 100755 --- a/test/functional/feature_backwards_compatibility.py +++ b/test/functional/feature_backwards_compatibility.py @@ -34,11 +34,12 @@ from test_framework.util import ( class BackwardsCompatibilityTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 9 + self.num_nodes = 10 # Add new version after each release: self.extra_args = [ ["-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # Pre-release: use to mine blocks. noban for immediate tx relay ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # Pre-release: use to receive coins, swap wallets, etc + ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v23.0 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v22.0 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v0.21.0 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v0.20.1 @@ -57,6 +58,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): self.add_nodes(self.num_nodes, extra_args=self.extra_args, versions=[ None, None, + 230000, 220000, 210000, 200100, diff --git a/test/functional/feature_blockfilterindex_prune.py b/test/functional/feature_blockfilterindex_prune.py deleted file mode 100755 index 2451988135..0000000000 --- a/test/functional/feature_blockfilterindex_prune.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2020-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test blockfilterindex in conjunction with prune.""" -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_equal, - assert_greater_than, - assert_raises_rpc_error, -) - - -class FeatureBlockfilterindexPruneTest(BitcoinTestFramework): - def set_test_params(self): - self.num_nodes = 1 - self.extra_args = [["-fastprune", "-prune=1", "-blockfilterindex=1"]] - - def sync_index(self, height): - expected = {'basic block filter index': {'synced': True, 'best_block_height': height}} - self.wait_until(lambda: self.nodes[0].getindexinfo() == expected) - - def run_test(self): - self.log.info("check if we can access a blockfilter when pruning is enabled but no blocks are actually pruned") - self.sync_index(height=200) - assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getbestblockhash())['filter']), 0) - self.generate(self.nodes[0], 500) - self.sync_index(height=700) - - self.log.info("prune some blocks") - pruneheight = self.nodes[0].pruneblockchain(400) - # the prune heights used here and below are magic numbers that are determined by the - # thresholds at which block files wrap, so they depend on disk serialization and default block file size. - assert_equal(pruneheight, 248) - - self.log.info("check if we can access the tips blockfilter when we have pruned some blocks") - assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getbestblockhash())['filter']), 0) - - self.log.info("check if we can access the blockfilter of a pruned block") - assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getblockhash(2))['filter']), 0) - - # mine and sync index up to a height that will later be the pruneheight - self.generate(self.nodes[0], 298) - self.sync_index(height=998) - - self.log.info("start node without blockfilterindex") - self.restart_node(0, extra_args=["-fastprune", "-prune=1"]) - - self.log.info("make sure accessing the blockfilters throws an error") - assert_raises_rpc_error(-1, "Index is not enabled for filtertype basic", self.nodes[0].getblockfilter, self.nodes[0].getblockhash(2)) - self.generate(self.nodes[0], 502) - - self.log.info("prune exactly up to the blockfilterindexes best block while blockfilters are disabled") - pruneheight_2 = self.nodes[0].pruneblockchain(1000) - assert_equal(pruneheight_2, 998) - self.restart_node(0, extra_args=["-fastprune", "-prune=1", "-blockfilterindex=1"]) - self.log.info("make sure that we can continue with the partially synced index after having pruned up to the index height") - self.sync_index(height=1500) - - self.log.info("prune below the blockfilterindexes best block while blockfilters are disabled") - self.restart_node(0, extra_args=["-fastprune", "-prune=1"]) - self.generate(self.nodes[0], 1000) - pruneheight_3 = self.nodes[0].pruneblockchain(2000) - assert_greater_than(pruneheight_3, pruneheight_2) - self.stop_node(0) - - self.log.info("make sure we get an init error when starting the node again with block filters") - self.nodes[0].assert_start_raises_init_error( - extra_args=["-fastprune", "-prune=1", "-blockfilterindex=1"], - expected_msg="Error: basic block filter index best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)", - ) - - self.log.info("make sure the node starts again with the -reindex arg") - self.start_node(0, extra_args=["-fastprune", "-prune=1", "-blockfilterindex", "-reindex"]) - - -if __name__ == '__main__': - FeatureBlockfilterindexPruneTest().main() diff --git a/test/functional/feature_coinstatsindex.py b/test/functional/feature_coinstatsindex.py index c70f8a83db..2e21638f80 100755 --- a/test/functional/feature_coinstatsindex.py +++ b/test/functional/feature_coinstatsindex.py @@ -18,9 +18,6 @@ from test_framework.blocktools import ( ) from test_framework.messages import ( COIN, - COutPoint, - CTransaction, - CTxIn, CTxOut, ) from test_framework.script import ( @@ -33,6 +30,11 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, ) +from test_framework.wallet import ( + MiniWallet, + getnewdestination, +) + class CoinStatsIndexTest(BitcoinTestFramework): def set_test_params(self): @@ -40,16 +42,12 @@ class CoinStatsIndexTest(BitcoinTestFramework): self.num_nodes = 2 self.supports_cli = False self.extra_args = [ - # Explicitly set the output type in order to have consistent tx vsize / fees - # for both legacy and descriptor wallets (disables the change address type detection algorithm) - ["-addresstype=bech32", "-changetype=bech32"], + [], ["-coinstatsindex"] ] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) self._test_coin_stats_index() self._test_use_index_option() self._test_reorg_index() @@ -69,9 +67,8 @@ class CoinStatsIndexTest(BitcoinTestFramework): index_hash_options = ['none', 'muhash'] # Generate a normal transaction and mine it - self.generate(node, COINBASE_MATURITY + 1) - address = self.nodes[0].get_deterministic_priv_key().address - node.sendtoaddress(address=address, amount=10, subtractfeefromamount=True) + self.generate(self.wallet, COINBASE_MATURITY + 1) + self.wallet.send_self_transfer(from_node=node) self.generate(node, 1) self.log.info("Test that gettxoutsetinfo() output is consistent with or without coinstatsindex option") @@ -136,36 +133,31 @@ class CoinStatsIndexTest(BitcoinTestFramework): assert_equal(res5['block_info'], { 'unspendable': 0, 'prevout_spent': 50, - 'new_outputs_ex_coinbase': Decimal('49.99995560'), - 'coinbase': Decimal('50.00004440'), + 'new_outputs_ex_coinbase': Decimal('49.99968800'), + 'coinbase': Decimal('50.00031200'), 'unspendables': { 'genesis_block': 0, 'bip30': 0, 'scripts': 0, - 'unclaimed_rewards': 0 + 'unclaimed_rewards': 0, } }) self.block_sanity_check(res5['block_info']) # Generate and send a normal tx with two outputs - tx1_inputs = [] - tx1_outputs = {self.nodes[0].getnewaddress(): 21, self.nodes[0].getnewaddress(): 42} - raw_tx1 = self.nodes[0].createrawtransaction(tx1_inputs, tx1_outputs) - funded_tx1 = self.nodes[0].fundrawtransaction(raw_tx1) - signed_tx1 = self.nodes[0].signrawtransactionwithwallet(funded_tx1['hex']) - tx1_txid = self.nodes[0].sendrawtransaction(signed_tx1['hex']) + tx1_txid, tx1_vout = self.wallet.send_to( + from_node=node, + scriptPubKey=self.wallet.get_scriptPubKey(), + amount=21 * COIN, + ) # Find the right position of the 21 BTC output - tx1_final = self.nodes[0].gettransaction(tx1_txid) - for output in tx1_final['details']: - if output['amount'] == Decimal('21.00000000') and output['category'] == 'receive': - n = output['vout'] + tx1_out_21 = self.wallet.get_utxo(txid=tx1_txid, vout=tx1_vout) # Generate and send another tx with an OP_RETURN output (which is unspendable) - tx2 = CTransaction() - tx2.vin.append(CTxIn(COutPoint(int(tx1_txid, 16), n), b'')) - tx2.vout.append(CTxOut(int(Decimal('20.99') * COIN), CScript([OP_RETURN] + [OP_FALSE]*30))) - tx2_hex = self.nodes[0].signrawtransactionwithwallet(tx2.serialize().hex())['hex'] + tx2 = self.wallet.create_self_transfer(utxo_to_spend=tx1_out_21)['tx'] + tx2.vout = [CTxOut(int(Decimal('20.99') * COIN), CScript([OP_RETURN] + [OP_FALSE] * 30))] + tx2_hex = tx2.serialize().hex() self.nodes[0].sendrawtransaction(tx2_hex) # Include both txs in a block @@ -177,14 +169,14 @@ class CoinStatsIndexTest(BitcoinTestFramework): assert_equal(res6['total_unspendable_amount'], Decimal('70.99000000')) assert_equal(res6['block_info'], { 'unspendable': Decimal('20.99000000'), - 'prevout_spent': 111, - 'new_outputs_ex_coinbase': Decimal('89.99993620'), - 'coinbase': Decimal('50.01006380'), + 'prevout_spent': 71, + 'new_outputs_ex_coinbase': Decimal('49.99999000'), + 'coinbase': Decimal('50.01001000'), 'unspendables': { 'genesis_block': 0, 'bip30': 0, 'scripts': Decimal('20.99000000'), - 'unclaimed_rewards': 0 + 'unclaimed_rewards': 0, } }) self.block_sanity_check(res6['block_info']) @@ -231,6 +223,22 @@ class CoinStatsIndexTest(BitcoinTestFramework): res10 = index_node.gettxoutsetinfo('muhash') assert(res8['txouts'] < res10['txouts']) + self.log.info("Test that the index works with -reindex") + + self.restart_node(1, extra_args=["-coinstatsindex", "-reindex"]) + res11 = index_node.gettxoutsetinfo('muhash') + assert_equal(res11, res10) + + self.log.info("Test that -reindex-chainstate is disallowed with coinstatsindex") + + self.stop_node(1) + self.nodes[1].assert_start_raises_init_error( + expected_msg='Error: -reindex-chainstate option is not compatible with -coinstatsindex. ' + 'Please temporarily disable coinstatsindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.', + extra_args=['-coinstatsindex', '-reindex-chainstate'], + ) + self.restart_node(1, extra_args=["-coinstatsindex"]) + def _test_use_index_option(self): self.log.info("Test use_index option for nodes running the index") @@ -246,7 +254,7 @@ class CoinStatsIndexTest(BitcoinTestFramework): # Generate two block, let the index catch up, then invalidate the blocks index_node = self.nodes[1] - reorg_blocks = self.generatetoaddress(index_node, 2, index_node.getnewaddress()) + reorg_blocks = self.generatetoaddress(index_node, 2, getnewdestination()[2]) reorg_block = reorg_blocks[1] res_invalid = index_node.gettxoutsetinfo('muhash') index_node.invalidateblock(reorg_blocks[0]) diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index eea5fa24ee..fe3196d5ee 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -247,7 +247,8 @@ class ConfArgsTest(BitcoinTestFramework): conf_file = os.path.join(default_data_dir, "bitcoin.conf") # datadir needs to be set before [chain] section - conf_file_contents = open(conf_file, encoding='utf8').read() + with open(conf_file, encoding='utf8') as f: + conf_file_contents = f.read() with open(conf_file, 'w', encoding='utf8') as f: f.write(f"datadir={new_data_dir}\n") f.write(conf_file_contents) diff --git a/test/functional/feature_csv_activation.py b/test/functional/feature_csv_activation.py index 6470c1c5eb..bff95c3b94 100755 --- a/test/functional/feature_csv_activation.py +++ b/test/functional/feature_csv_activation.py @@ -112,6 +112,7 @@ class BIP68_112_113Test(BitcoinTestFramework): tx.nVersion = txversion self.miniwallet.sign_tx(tx) tx.vin[0].scriptSig = CScript([-1, OP_CHECKSEQUENCEVERIFY, OP_DROP] + list(CScript(tx.vin[0].scriptSig))) + tx.rehash() return tx def create_bip112emptystack(self, input, txversion): @@ -119,6 +120,7 @@ class BIP68_112_113Test(BitcoinTestFramework): tx.nVersion = txversion self.miniwallet.sign_tx(tx) tx.vin[0].scriptSig = CScript([OP_CHECKSEQUENCEVERIFY] + list(CScript(tx.vin[0].scriptSig))) + tx.rehash() return tx def send_generic_input_tx(self, coinbases): @@ -136,7 +138,6 @@ class BIP68_112_113Test(BitcoinTestFramework): tx.nVersion = txversion tx.vin[0].nSequence = locktime + locktime_delta self.miniwallet.sign_tx(tx) - tx.rehash() txs.append({'tx': tx, 'sdf': sdf, 'stf': stf}) return txs @@ -339,20 +340,16 @@ class BIP68_112_113Test(BitcoinTestFramework): # BIP 113 tests should now fail regardless of version number if nLockTime isn't satisfied by new rules bip113tx_v1.nLockTime = self.last_block_time - 600 * 5 # = MTP of prior block (not <) but < time put on current block self.miniwallet.sign_tx(bip113tx_v1) - bip113tx_v1.rehash() bip113tx_v2.nLockTime = self.last_block_time - 600 * 5 # = MTP of prior block (not <) but < time put on current block self.miniwallet.sign_tx(bip113tx_v2) - bip113tx_v2.rehash() for bip113tx in [bip113tx_v1, bip113tx_v2]: self.send_blocks([self.create_test_block([bip113tx])], success=False, reject_reason='bad-txns-nonfinal') # BIP 113 tests should now pass if the locktime is < MTP bip113tx_v1.nLockTime = self.last_block_time - 600 * 5 - 1 # < MTP of prior block self.miniwallet.sign_tx(bip113tx_v1) - bip113tx_v1.rehash() bip113tx_v2.nLockTime = self.last_block_time - 600 * 5 - 1 # < MTP of prior block self.miniwallet.sign_tx(bip113tx_v2) - bip113tx_v2.rehash() for bip113tx in [bip113tx_v1, bip113tx_v2]: self.send_blocks([self.create_test_block([bip113tx])]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) @@ -477,7 +474,6 @@ class BIP68_112_113Test(BitcoinTestFramework): for tx in [tx['tx'] for tx in bip112txs_vary_OP_CSV_v2 if not tx['sdf'] and tx['stf']]: tx.vin[0].nSequence = BASE_RELATIVE_LOCKTIME | SEQ_TYPE_FLAG self.miniwallet.sign_tx(tx) - tx.rehash() time_txs.append(tx) self.send_blocks([self.create_test_block(time_txs)]) diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index 233ffd60da..422612a78e 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -3,25 +3,13 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test fee estimation code.""" +from copy import deepcopy from decimal import Decimal import os import random from test_framework.messages import ( COIN, - COutPoint, - CTransaction, - CTxIn, - CTxOut, -) -from test_framework.script import ( - CScript, - OP_1, - OP_DROP, - OP_TRUE, -) -from test_framework.script_util import ( - script_to_p2sh_script, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -31,22 +19,14 @@ from test_framework.util import ( assert_raises_rpc_error, satoshi_round, ) - -# Construct 2 trivial P2SH's and the ScriptSigs that spend them -# So we can create many transactions without needing to spend -# time signing. -SCRIPT = CScript([OP_1, OP_DROP]) -P2SH = script_to_p2sh_script(SCRIPT) -REDEEM_SCRIPT = CScript([OP_TRUE, SCRIPT]) +from test_framework.wallet import MiniWallet def small_txpuzzle_randfee( - from_node, conflist, unconflist, amount, min_fee, fee_increment + wallet, from_node, conflist, unconflist, amount, min_fee, fee_increment ): - """Create and send a transaction with a random fee. + """Create and send a transaction with a random fee using MiniWallet. - The transaction pays to a trivial P2SH script, and assumes that its inputs - are of the same form. The function takes a list of confirmed outputs and unconfirmed outputs and attempts to use the confirmed list first for its inputs. It adds the newly created outputs to the unconfirmed list. @@ -58,23 +38,29 @@ def small_txpuzzle_randfee( rand_fee = float(fee_increment) * (1.1892 ** random.randint(0, 28)) # Total fee ranges from min_fee to min_fee + 127*fee_increment fee = min_fee - fee_increment + satoshi_round(rand_fee) - tx = CTransaction() + utxos_to_spend = [] total_in = Decimal("0.00000000") while total_in <= (amount + fee) and len(conflist) > 0: t = conflist.pop(0) - total_in += t["amount"] - tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), REDEEM_SCRIPT)) + total_in += t["value"] + utxos_to_spend.append(t) while total_in <= (amount + fee) and len(unconflist) > 0: t = unconflist.pop(0) - total_in += t["amount"] - tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), REDEEM_SCRIPT)) + total_in += t["value"] + utxos_to_spend.append(t) if total_in <= amount + fee: raise RuntimeError(f"Insufficient funds: need {amount + fee}, have {total_in}") - tx.vout.append(CTxOut(int((total_in - amount - fee) * COIN), P2SH)) - tx.vout.append(CTxOut(int(amount * COIN), P2SH)) + tx = wallet.create_self_transfer_multi( + from_node=from_node, + utxos_to_spend=utxos_to_spend, + fee_per_output=0) + tx.vout[0].nValue = int((total_in - amount - fee) * COIN) + tx.vout.append(deepcopy(tx.vout[0])) + tx.vout[1].nValue = int(amount * COIN) + txid = from_node.sendrawtransaction(hexstring=tx.serialize().hex(), maxfeerate=0) - unconflist.append({"txid": txid, "vout": 0, "amount": total_in - amount - fee}) - unconflist.append({"txid": txid, "vout": 1, "amount": amount}) + unconflist.append({"txid": txid, "vout": 0, "value": total_in - amount - fee}) + unconflist.append({"txid": txid, "vout": 1, "value": amount}) return (tx.serialize().hex(), fee) @@ -129,17 +115,13 @@ def check_estimates(node, fees_seen): check_smart_estimates(node, fees_seen) -def send_tx(node, utxo, feerate): +def send_tx(wallet, node, utxo, feerate): """Broadcast a 1in-1out transaction with a specific input and feerate (sat/vb).""" - tx = CTransaction() - tx.vin = [CTxIn(COutPoint(int(utxo["txid"], 16), utxo["vout"]), REDEEM_SCRIPT)] - tx.vout = [CTxOut(int(utxo["amount"] * COIN), P2SH)] - - # vbytes == bytes as we are using legacy transactions - fee = tx.get_vsize() * feerate - tx.vout[0].nValue -= fee - - return node.sendrawtransaction(tx.serialize().hex()) + return wallet.send_self_transfer( + from_node=node, + utxo_to_spend=utxo, + fee_rate=Decimal(feerate * 1000) / COIN, + )['txid'] class EstimateFeeTest(BitcoinTestFramework): @@ -152,9 +134,6 @@ class EstimateFeeTest(BitcoinTestFramework): ["-whitelist=noban@127.0.0.1", "-blockmaxweight=32000"], ] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def setup_network(self): """ We'll setup the network to have 3 nodes that all mine with different parameters. @@ -168,9 +147,6 @@ class EstimateFeeTest(BitcoinTestFramework): # (68k weight is room enough for 120 or so transactions) # Node2 is a stingy miner, that # produces too small blocks (room for only 55 or so transactions) - self.start_nodes() - self.import_deterministic_coinbase_privkeys() - self.stop_nodes() def transact_and_mine(self, numblocks, mining_node): min_fee = Decimal("0.00001") @@ -183,6 +159,7 @@ class EstimateFeeTest(BitcoinTestFramework): for _ in range(random.randrange(100 - 50, 100 + 50)): from_index = random.randint(1, 2) (txhex, fee) = small_txpuzzle_randfee( + self.wallet, self.nodes[from_index], self.confutxo, self.memutxo, @@ -205,24 +182,10 @@ class EstimateFeeTest(BitcoinTestFramework): def initial_split(self, node): """Split two coinbase UTxOs into many small coins""" - utxo_count = 2048 - self.confutxo = [] - splitted_amount = Decimal("0.04") - fee = Decimal("0.1") - change = Decimal("100") - splitted_amount * utxo_count - fee - tx = CTransaction() - tx.vin = [ - CTxIn(COutPoint(int(cb["txid"], 16), cb["vout"])) - for cb in node.listunspent()[:2] - ] - tx.vout = [CTxOut(int(splitted_amount * COIN), P2SH) for _ in range(utxo_count)] - tx.vout.append(CTxOut(int(change * COIN), P2SH)) - txhex = node.signrawtransactionwithwallet(tx.serialize().hex())["hex"] - txid = node.sendrawtransaction(txhex) - self.confutxo = [ - {"txid": txid, "vout": i, "amount": splitted_amount} - for i in range(utxo_count) - ] + self.confutxo = self.wallet.send_self_transfer_multi( + from_node=node, + utxos_to_spend=[self.wallet.get_utxo() for _ in range(2)], + num_outputs=2048)['new_utxos'] while len(node.getrawmempool()) > 0: self.generate(node, 1, sync_fun=self.no_op) @@ -284,12 +247,12 @@ class EstimateFeeTest(BitcoinTestFramework): # Broadcast 45 low fee transactions that will need to be RBF'd for _ in range(45): u = utxos.pop(0) - txid = send_tx(node, u, low_feerate) + txid = send_tx(self.wallet, node, u, low_feerate) utxos_to_respend.append(u) txids_to_replace.append(txid) # Broadcast 5 low fee transaction which don't need to for _ in range(5): - send_tx(node, utxos.pop(0), low_feerate) + send_tx(self.wallet, node, utxos.pop(0), low_feerate) # Mine the transactions on another node self.sync_mempools(wait=0.1, nodes=[node, miner]) for txid in txids_to_replace: @@ -298,7 +261,7 @@ class EstimateFeeTest(BitcoinTestFramework): # RBF the low-fee transactions while len(utxos_to_respend) > 0: u = utxos_to_respend.pop(0) - send_tx(node, u, high_feerate) + send_tx(self.wallet, node, u, high_feerate) # Mine the last replacement txs self.sync_mempools(wait=0.1, nodes=[node, miner]) @@ -316,6 +279,8 @@ class EstimateFeeTest(BitcoinTestFramework): # Split two coinbases into many small utxos self.start_node(0) + self.wallet = MiniWallet(self.nodes[0]) + self.wallet.rescan_utxos() self.initial_split(self.nodes[0]) self.log.info("Finished splitting") diff --git a/test/functional/feature_index_prune.py b/test/functional/feature_index_prune.py new file mode 100755 index 0000000000..2bf57db923 --- /dev/null +++ b/test/functional/feature_index_prune.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test indices in conjunction with prune.""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_rpc_error, + p2p_port, +) + + +class FeatureIndexPruneTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 4 + self.extra_args = [ + ["-fastprune", "-prune=1", "-blockfilterindex=1"], + ["-fastprune", "-prune=1", "-coinstatsindex=1"], + ["-fastprune", "-prune=1", "-blockfilterindex=1", "-coinstatsindex=1"], + [] + ] + + def sync_index(self, height): + expected_filter = { + 'basic block filter index': {'synced': True, 'best_block_height': height}, + } + self.wait_until(lambda: self.nodes[0].getindexinfo() == expected_filter) + + expected_stats = { + 'coinstatsindex': {'synced': True, 'best_block_height': height} + } + self.wait_until(lambda: self.nodes[1].getindexinfo() == expected_stats) + + expected = {**expected_filter, **expected_stats} + self.wait_until(lambda: self.nodes[2].getindexinfo() == expected) + + def reconnect_nodes(self): + self.connect_nodes(0,1) + self.connect_nodes(0,2) + self.connect_nodes(0,3) + + def mine_batches(self, blocks): + n = blocks // 250 + for _ in range(n): + self.generate(self.nodes[0], 250) + self.generate(self.nodes[0], blocks % 250) + self.sync_blocks() + + def restart_without_indices(self): + for i in range(3): + self.restart_node(i, extra_args=["-fastprune", "-prune=1"]) + self.reconnect_nodes() + + def run_test(self): + filter_nodes = [self.nodes[0], self.nodes[2]] + stats_nodes = [self.nodes[1], self.nodes[2]] + + self.log.info("check if we can access blockfilters and coinstats when pruning is enabled but no blocks are actually pruned") + self.sync_index(height=200) + tip = self.nodes[0].getbestblockhash() + for node in filter_nodes: + assert_greater_than(len(node.getblockfilter(tip)['filter']), 0) + for node in stats_nodes: + assert(node.gettxoutsetinfo(hash_type="muhash", hash_or_height=tip)['muhash']) + + self.mine_batches(500) + self.sync_index(height=700) + + self.log.info("prune some blocks") + for node in self.nodes[:2]: + with node.assert_debug_log(['limited pruning to height 689']): + pruneheight_new = node.pruneblockchain(400) + # the prune heights used here and below are magic numbers that are determined by the + # thresholds at which block files wrap, so they depend on disk serialization and default block file size. + assert_equal(pruneheight_new, 249) + + self.log.info("check if we can access the tips blockfilter and coinstats when we have pruned some blocks") + tip = self.nodes[0].getbestblockhash() + for node in filter_nodes: + assert_greater_than(len(node.getblockfilter(tip)['filter']), 0) + for node in stats_nodes: + assert(node.gettxoutsetinfo(hash_type="muhash", hash_or_height=tip)['muhash']) + + self.log.info("check if we can access the blockfilter and coinstats of a pruned block") + height_hash = self.nodes[0].getblockhash(2) + for node in filter_nodes: + assert_greater_than(len(node.getblockfilter(height_hash)['filter']), 0) + for node in stats_nodes: + assert(node.gettxoutsetinfo(hash_type="muhash", hash_or_height=height_hash)['muhash']) + + # mine and sync index up to a height that will later be the pruneheight + self.generate(self.nodes[0], 51) + self.sync_index(height=751) + + self.restart_without_indices() + + self.log.info("make sure trying to access the indices throws errors") + for node in filter_nodes: + msg = "Index is not enabled for filtertype basic" + assert_raises_rpc_error(-1, msg, node.getblockfilter, height_hash) + for node in stats_nodes: + msg = "Querying specific block heights requires coinstatsindex" + assert_raises_rpc_error(-8, msg, node.gettxoutsetinfo, "muhash", height_hash) + + self.mine_batches(749) + + self.log.info("prune exactly up to the indices best blocks while the indices are disabled") + for i in range(3): + pruneheight_2 = self.nodes[i].pruneblockchain(1000) + assert_equal(pruneheight_2, 751) + # Restart the nodes again with the indices activated + self.restart_node(i, extra_args=self.extra_args[i]) + + self.log.info("make sure that we can continue with the partially synced indices after having pruned up to the index height") + self.sync_index(height=1500) + + self.log.info("prune further than the indices best blocks while the indices are disabled") + self.restart_without_indices() + self.mine_batches(1000) + + for i in range(3): + pruneheight_3 = self.nodes[i].pruneblockchain(2000) + assert_greater_than(pruneheight_3, pruneheight_2) + self.stop_node(i) + + self.log.info("make sure we get an init error when starting the nodes again with the indices") + filter_msg = "Error: basic block filter index best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)" + stats_msg = "Error: coinstatsindex best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)" + for i, msg in enumerate([filter_msg, stats_msg, filter_msg]): + self.nodes[i].assert_start_raises_init_error(extra_args=self.extra_args[i], expected_msg=msg) + + self.log.info("make sure the nodes start again with the indices and an additional -reindex arg") + ip_port = "127.0.0.1:" + str(p2p_port(3)) + for i in range(3): + # The nodes need to be reconnected to the non-pruning node upon restart, otherwise they will be stuck + restart_args = self.extra_args[i]+["-reindex", f"-connect={ip_port}"] + self.restart_node(i, extra_args=restart_args) + + self.sync_blocks(timeout=300) + + for node in self.nodes[:2]: + with node.assert_debug_log(['limited pruning to height 2489']): + pruneheight_new = node.pruneblockchain(2500) + assert_equal(pruneheight_new, 2006) + + self.log.info("ensure that prune locks don't prevent indices from failing in a reorg scenario") + with self.nodes[0].assert_debug_log(['basic block filter index prune lock moved back to 2480']): + self.nodes[3].invalidateblock(self.nodes[0].getblockhash(2480)) + self.generate(self.nodes[3], 30) + self.sync_blocks() + + +if __name__ == '__main__': + FeatureIndexPruneTest().main() diff --git a/test/functional/feature_maxuploadtarget.py b/test/functional/feature_maxuploadtarget.py index 24f79dda67..0b9d651226 100755 --- a/test/functional/feature_maxuploadtarget.py +++ b/test/functional/feature_maxuploadtarget.py @@ -13,10 +13,19 @@ if uploadtarget has been reached. from collections import defaultdict import time -from test_framework.messages import CInv, MSG_BLOCK, msg_getdata +from test_framework.messages import ( + CInv, + MSG_BLOCK, + msg_getdata, +) from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, mine_large_block +from test_framework.util import ( + assert_equal, + mine_large_block, +) +from test_framework.wallet import MiniWallet + class TestP2PConn(P2PInterface): def __init__(self): @@ -41,12 +50,6 @@ class MaxUploadTest(BitcoinTestFramework): ]] self.supports_cli = False - # Cache for utxos, as the listunspent may take a long time later in the test - self.utxo_cache = [] - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def run_test(self): # Before we connect anything, we first set the time on the node # to be in the past, otherwise things break because the CNode @@ -55,7 +58,8 @@ class MaxUploadTest(BitcoinTestFramework): self.nodes[0].setmocktime(old_time) # Generate some old blocks - self.generate(self.nodes[0], 130) + self.wallet = MiniWallet(self.nodes[0]) + self.generate(self.wallet, 130) # p2p_conns[0] will only request old blocks # p2p_conns[1] will only request new blocks @@ -66,7 +70,7 @@ class MaxUploadTest(BitcoinTestFramework): p2p_conns.append(self.nodes[0].add_p2p_connection(TestP2PConn())) # Now mine a big block - mine_large_block(self, self.nodes[0], self.utxo_cache) + mine_large_block(self, self.wallet, self.nodes[0]) # Store the hash; we'll request this later big_old_block = self.nodes[0].getbestblockhash() @@ -77,7 +81,7 @@ class MaxUploadTest(BitcoinTestFramework): self.nodes[0].setmocktime(int(time.time()) - 2*60*60*24) # Mine one more block, so that the prior block looks old - mine_large_block(self, self.nodes[0], self.utxo_cache) + mine_large_block(self, self.wallet, self.nodes[0]) # We'll be requesting this new block too big_new_block = self.nodes[0].getbestblockhash() diff --git a/test/functional/feature_proxy.py b/test/functional/feature_proxy.py index fb0f6d7cb7..8541c3ed88 100755 --- a/test/functional/feature_proxy.py +++ b/test/functional/feature_proxy.py @@ -30,6 +30,12 @@ addnode connect to generic DNS name addnode connect to a CJDNS address - Test getnetworkinfo for each node + +- Test passing invalid -proxy +- Test passing invalid -onion +- Test passing invalid -i2psam +- Test passing -onlynet=onion without -proxy or -onion +- Test passing -onlynet=onion with -onion=0 and with -noonion """ import socket @@ -234,7 +240,15 @@ class ProxyTest(BitcoinTestFramework): return r self.log.info("Test RPC getnetworkinfo") - n0 = networks_dict(self.nodes[0].getnetworkinfo()) + nodes_network_info = [] + + self.log.debug("Test that setting -proxy disables local address discovery, i.e. -discover=0") + for node in self.nodes: + network_info = node.getnetworkinfo() + assert_equal(network_info["localaddresses"], []) + nodes_network_info.append(network_info) + + n0 = networks_dict(nodes_network_info[0]) assert_equal(NETWORKS, n0.keys()) for net in NETWORKS: if net == NET_I2P: @@ -249,7 +263,7 @@ class ProxyTest(BitcoinTestFramework): assert_equal(n0['i2p']['reachable'], False) assert_equal(n0['cjdns']['reachable'], False) - n1 = networks_dict(self.nodes[1].getnetworkinfo()) + n1 = networks_dict(nodes_network_info[1]) assert_equal(NETWORKS, n1.keys()) for net in ['ipv4', 'ipv6']: assert_equal(n1[net]['proxy'], f'{self.conf1.addr[0]}:{self.conf1.addr[1]}') @@ -261,14 +275,15 @@ class ProxyTest(BitcoinTestFramework): assert_equal(n1['i2p']['proxy_randomize_credentials'], False) assert_equal(n1['i2p']['reachable'], True) - n2 = networks_dict(self.nodes[2].getnetworkinfo()) + n2 = networks_dict(nodes_network_info[2]) assert_equal(NETWORKS, n2.keys()) + proxy = f'{self.conf2.addr[0]}:{self.conf2.addr[1]}' for net in NETWORKS: if net == NET_I2P: expected_proxy = '' expected_randomize = False else: - expected_proxy = f'{self.conf2.addr[0]}:{self.conf2.addr[1]}' + expected_proxy = proxy expected_randomize = True assert_equal(n2[net]['proxy'], expected_proxy) assert_equal(n2[net]['proxy_randomize_credentials'], expected_randomize) @@ -277,20 +292,18 @@ class ProxyTest(BitcoinTestFramework): assert_equal(n2['cjdns']['reachable'], False) if self.have_ipv6: - n3 = networks_dict(self.nodes[3].getnetworkinfo()) + n3 = networks_dict(nodes_network_info[3]) assert_equal(NETWORKS, n3.keys()) + proxy = f'[{self.conf3.addr[0]}]:{self.conf3.addr[1]}' for net in NETWORKS: - if net == NET_I2P or net == NET_ONION: - expected_proxy = '' - else: - expected_proxy = f'[{self.conf3.addr[0]}]:{self.conf3.addr[1]}' + expected_proxy = '' if net == NET_I2P or net == NET_ONION else proxy assert_equal(n3[net]['proxy'], expected_proxy) assert_equal(n3[net]['proxy_randomize_credentials'], False) assert_equal(n3['onion']['reachable'], False) assert_equal(n3['i2p']['reachable'], False) assert_equal(n3['cjdns']['reachable'], False) - n4 = networks_dict(self.nodes[4].getnetworkinfo()) + n4 = networks_dict(nodes_network_info[4]) assert_equal(NETWORKS, n4.keys()) for net in NETWORKS: if net == NET_I2P: @@ -305,6 +318,37 @@ class ProxyTest(BitcoinTestFramework): assert_equal(n4['i2p']['reachable'], False) assert_equal(n4['cjdns']['reachable'], True) + self.stop_node(1) + + self.log.info("Test passing invalid -proxy raises expected init error") + self.nodes[1].extra_args = ["-proxy=abc:def"] + msg = "Error: Invalid -proxy address or hostname: 'abc:def'" + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + + self.log.info("Test passing invalid -onion raises expected init error") + self.nodes[1].extra_args = ["-onion=xyz:abc"] + msg = "Error: Invalid -onion address or hostname: 'xyz:abc'" + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + + self.log.info("Test passing invalid -i2psam raises expected init error") + self.nodes[1].extra_args = ["-i2psam=def:xyz"] + msg = "Error: Invalid -i2psam address or hostname: 'def:xyz'" + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + + msg = ( + "Error: Outbound connections restricted to Tor (-onlynet=onion) but " + "the proxy for reaching the Tor network is not provided (no -proxy= " + "and no -onion= given) or it is explicitly forbidden (-onion=0)" + ) + self.log.info("Test passing -onlynet=onion without -proxy or -onion raises expected init error") + self.nodes[1].extra_args = ["-onlynet=onion"] + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + + self.log.info("Test passing -onlynet=onion with -onion=0/-noonion raises expected init error") + for arg in ["-onion=0", "-noonion"]: + self.nodes[1].extra_args = ["-onlynet=onion", arg] + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + if __name__ == '__main__': ProxyTest().main() diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index ba3c5053cb..77524e85a3 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -125,6 +125,7 @@ class PruneTest(BitcoinTestFramework): self.sync_blocks(self.nodes[0:5]) def test_invalid_command_line_options(self): + self.stop_node(0) self.nodes[0].assert_start_raises_init_error( expected_msg='Error: Prune cannot be configured with a negative value.', extra_args=['-prune=-1'], @@ -138,8 +139,8 @@ class PruneTest(BitcoinTestFramework): extra_args=['-prune=550', '-txindex'], ) self.nodes[0].assert_start_raises_init_error( - expected_msg='Error: Prune mode is incompatible with -coinstatsindex.', - extra_args=['-prune=550', '-coinstatsindex'], + expected_msg='Error: Prune mode is incompatible with -reindex-chainstate. Use full -reindex instead.', + extra_args=['-prune=550', '-reindex-chainstate'], ) def test_height_min(self): diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 6d7f1def88..f0faf1421b 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -86,18 +86,18 @@ class SegWitTest(BitcoinTestFramework): [ "-acceptnonstdtxn=1", "-rpcserialversion=0", - "-testactivationheight=segwit@432", + "-testactivationheight=segwit@165", "-addresstype=legacy", ], [ "-acceptnonstdtxn=1", "-rpcserialversion=1", - "-testactivationheight=segwit@432", + "-testactivationheight=segwit@165", "-addresstype=legacy", ], [ "-acceptnonstdtxn=1", - "-testactivationheight=segwit@432", + "-testactivationheight=segwit@165", "-addresstype=legacy", ], ] @@ -117,12 +117,6 @@ class SegWitTest(BitcoinTestFramework): assert_equal(len(node.getblock(block[0])["tx"]), 2) self.sync_blocks() - def skip_mine(self, node, txid, sign, redeem_script=""): - send_to_witness(1, node, getutxo(txid), self.pubkey[0], False, Decimal("49.998"), sign, redeem_script) - block = self.generate(node, 1) - assert_equal(len(node.getblock(block[0])["tx"]), 1) - self.sync_blocks() - def fail_accept(self, node, error_msg, txid, sign, redeem_script=""): assert_raises_rpc_error(-26, error_msg, send_to_witness, use_p2wsh=1, node=node, utxo=getutxo(txid), pubkey=self.pubkey[0], encode_p2sh=False, amount=Decimal("49.998"), sign=sign, insert_redeem_script=redeem_script) @@ -197,23 +191,21 @@ class SegWitTest(BitcoinTestFramework): assert_equal(self.nodes[1].getbalance(), 20 * Decimal("49.999")) assert_equal(self.nodes[2].getbalance(), 20 * Decimal("49.999")) - self.generate(self.nodes[0], 260) # block 423 - - self.log.info("Verify witness txs are skipped for mining before the fork") - self.skip_mine(self.nodes[2], wit_ids[NODE_2][P2WPKH][0], True) # block 424 - self.skip_mine(self.nodes[2], wit_ids[NODE_2][P2WSH][0], True) # block 425 - self.skip_mine(self.nodes[2], p2sh_ids[NODE_2][P2WPKH][0], True) # block 426 - self.skip_mine(self.nodes[2], p2sh_ids[NODE_2][P2WSH][0], True) # block 427 - self.log.info("Verify unsigned p2sh witness txs without a redeem script are invalid") self.fail_accept(self.nodes[2], "mandatory-script-verify-flag-failed (Operation not valid with the current stack size)", p2sh_ids[NODE_2][P2WPKH][1], sign=False) self.fail_accept(self.nodes[2], "mandatory-script-verify-flag-failed (Operation not valid with the current stack size)", p2sh_ids[NODE_2][P2WSH][1], sign=False) - self.generate(self.nodes[2], 4) # blocks 428-431 + self.generate(self.nodes[0], 1) # block 164 + + self.log.info("Verify witness txs are mined as soon as segwit activates") + + send_to_witness(1, self.nodes[2], getutxo(wit_ids[NODE_2][P2WPKH][0]), self.pubkey[0], encode_p2sh=False, amount=Decimal("49.998"), sign=True) + send_to_witness(1, self.nodes[2], getutxo(wit_ids[NODE_2][P2WSH][0]), self.pubkey[0], encode_p2sh=False, amount=Decimal("49.998"), sign=True) + send_to_witness(1, self.nodes[2], getutxo(p2sh_ids[NODE_2][P2WPKH][0]), self.pubkey[0], encode_p2sh=False, amount=Decimal("49.998"), sign=True) + send_to_witness(1, self.nodes[2], getutxo(p2sh_ids[NODE_2][P2WSH][0]), self.pubkey[0], encode_p2sh=False, amount=Decimal("49.998"), sign=True) - self.log.info("Verify previous witness txs skipped for mining can now be mined") assert_equal(len(self.nodes[2].getrawmempool()), 4) - blockhash = self.generate(self.nodes[2], 1)[0] # block 432 (first block with new rules; 432 = 144 * 3) + blockhash = self.generate(self.nodes[2], 1)[0] # block 165 (first block with new rules) assert_equal(len(self.nodes[2].getrawmempool()), 0) segwit_tx_list = self.nodes[2].getblock(blockhash)["tx"] assert_equal(len(segwit_tx_list), 5) @@ -255,10 +247,10 @@ class SegWitTest(BitcoinTestFramework): self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program was passed an empty witness)', p2sh_ids[NODE_2][P2WSH][2], sign=False, redeem_script=witness_script(True, self.pubkey[2])) self.log.info("Verify default node can now use witness txs") - self.success_mine(self.nodes[0], wit_ids[NODE_0][P2WPKH][0], True) # block 432 - self.success_mine(self.nodes[0], wit_ids[NODE_0][P2WSH][0], True) # block 433 - self.success_mine(self.nodes[0], p2sh_ids[NODE_0][P2WPKH][0], True) # block 434 - self.success_mine(self.nodes[0], p2sh_ids[NODE_0][P2WSH][0], True) # block 435 + self.success_mine(self.nodes[0], wit_ids[NODE_0][P2WPKH][0], True) + self.success_mine(self.nodes[0], wit_ids[NODE_0][P2WSH][0], True) + self.success_mine(self.nodes[0], p2sh_ids[NODE_0][P2WPKH][0], True) + self.success_mine(self.nodes[0], p2sh_ids[NODE_0][P2WSH][0], True) self.log.info("Verify sigops are counted in GBT with BIP141 rules after the fork") txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index 3e3d4b3c77..0e44038196 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -10,7 +10,6 @@ from test_framework.blocktools import ( create_block, add_witness_commitment, MAX_BLOCK_SIGOPS_WEIGHT, - NORMAL_GBT_REQUEST_PARAMS, WITNESS_SCALE_FACTOR, ) from test_framework.messages import ( @@ -96,10 +95,9 @@ from test_framework.util import assert_raises_rpc_error, assert_equal from test_framework.key import generate_privkey, compute_xonly_pubkey, sign_schnorr, tweak_add_privkey, ECKey from test_framework.address import ( hash160, - program_to_witness + program_to_witness, ) from collections import OrderedDict, namedtuple -from enum import Enum from io import BytesIO import json import hashlib @@ -458,7 +456,7 @@ def spend(tx, idx, utxos, **kwargs): # Each spender is a tuple of: # - A scriptPubKey which is to be spent from (CScript) # - A comment describing the test (string) -# - Whether the spending (on itself) is expected to be standard (Enum.Standard) +# - Whether the spending (on itself) is expected to be standard (bool) # - A tx-signing lambda returning (scriptsig, witness_stack), taking as inputs: # - A transaction to sign (CTransaction) # - An input position (int) @@ -470,14 +468,9 @@ def spend(tx, idx, utxos, **kwargs): # - Whether this test demands being placed in a txin with no corresponding txout (for testing SIGHASH_SINGLE behavior) Spender = namedtuple("Spender", "script,comment,is_standard,sat_function,err_msg,sigops_weight,no_fail,need_vin_vout_mismatch") -# The full node versions that treat the tx standard. -# ALL means any version -# V23 means the major version 23.0 and any later version -# NONE means no version -Standard = Enum('Standard', 'ALL V23 NONE') -def make_spender(comment, *, tap=None, witv0=False, script=None, pkh=None, p2sh=False, spk_mutate_pre_p2sh=None, failure=None, standard=Standard.ALL, err_msg=None, sigops_weight=0, need_vin_vout_mismatch=False, **kwargs): +def make_spender(comment, *, tap=None, witv0=False, script=None, pkh=None, p2sh=False, spk_mutate_pre_p2sh=None, failure=None, standard=True, err_msg=None, sigops_weight=0, need_vin_vout_mismatch=False, **kwargs): """Helper for constructing Spender objects using the context signing framework. * tap: a TaprootInfo object (see taproot_construct), for Taproot spends (cannot be combined with pkh, witv0, or script) @@ -487,18 +480,13 @@ def make_spender(comment, *, tap=None, witv0=False, script=None, pkh=None, p2sh= * p2sh: whether the output is P2SH wrapper (this is supported even for Taproot, where it makes the output unencumbered) * spk_mutate_pre_psh: a callable to be applied to the script (before potentially P2SH-wrapping it) * failure: a dict of entries to override in the context when intentionally failing to spend (if None, no_fail will be set) - * standard: whether the (valid version of) spending is expected to be standard (True is mapped to Standard.ALL, False is mapped to Standard.NONE) + * standard: whether the (valid version of) spending is expected to be standard * err_msg: a string with an expected error message for failure (or None, if not cared about) * sigops_weight: the pre-taproot sigops weight consumed by a successful spend * need_vin_vout_mismatch: whether this test requires being tested in a transaction input that has no corresponding transaction output. """ - if standard == True: - standard = Standard.ALL - elif standard == False: - standard = Standard.NONE - conf = dict() # Compute scriptPubKey and set useful defaults based on the inputs. @@ -1168,29 +1156,21 @@ def spenders_taproot_active(): return spenders -def spenders_taproot_inactive(): - """Spenders for testing that pre-activation Taproot rules don't apply.""" + +def spenders_taproot_nonstandard(): + """Spenders for testing that post-activation Taproot rules may be nonstandard.""" spenders = [] sec = generate_privkey() pub, _ = compute_xonly_pubkey(sec) scripts = [ - ("pk", CScript([pub, OP_CHECKSIG])), ("future_leaf", CScript([pub, OP_CHECKSIG]), 0xc2), ("op_success", CScript([pub, OP_CHECKSIG, OP_0, OP_IF, CScriptOp(0x50), OP_ENDIF])), ] tap = taproot_construct(pub, scripts) - # Test that keypath spending is valid & non-standard, regardless of validity. - add_spender(spenders, "inactive/keypath_valid", key=sec, tap=tap, standard=Standard.V23) - add_spender(spenders, "inactive/keypath_invalidsig", key=sec, tap=tap, standard=False, sighash=bitflipper(default_sighash)) - add_spender(spenders, "inactive/keypath_empty", key=sec, tap=tap, standard=False, witness=[]) - - # Same for scriptpath spending (and features like annex, leaf versions, or OP_SUCCESS don't change this) - add_spender(spenders, "inactive/scriptpath_valid", key=sec, tap=tap, leaf="pk", standard=Standard.V23, inputs=[getter("sign")]) - add_spender(spenders, "inactive/scriptpath_invalidsig", key=sec, tap=tap, leaf="pk", standard=False, inputs=[getter("sign")], sighash=bitflipper(default_sighash)) - add_spender(spenders, "inactive/scriptpath_invalidcb", key=sec, tap=tap, leaf="pk", standard=False, inputs=[getter("sign")], controlblock=bitflipper(default_controlblock)) + # Test that features like annex, leaf versions, or OP_SUCCESS are valid but non-standard add_spender(spenders, "inactive/scriptpath_valid_unkleaf", key=sec, tap=tap, leaf="future_leaf", standard=False, inputs=[getter("sign")]) add_spender(spenders, "inactive/scriptpath_invalid_unkleaf", key=sec, tap=tap, leaf="future_leaf", standard=False, inputs=[getter("sign")], sighash=bitflipper(default_sighash)) add_spender(spenders, "inactive/scriptpath_valid_opsuccess", key=sec, tap=tap, leaf="op_success", standard=False, inputs=[getter("sign")]) @@ -1218,7 +1198,7 @@ def dump_json_test(tx, input_utxos, idx, success, failure): # The "final" field indicates that a spend should be always valid, even with more validation flags enabled # than the listed ones. Use standardness as a proxy for this (which gives a conservative underestimate). - if spender.is_standard == Standard.ALL: + if spender.is_standard: fields.append(("final", True)) def dump_witness(wit): @@ -1245,31 +1225,14 @@ class TaprootTest(BitcoinTestFramework): def add_options(self, parser): parser.add_argument("--dumptests", dest="dump_tests", default=False, action="store_true", help="Dump generated test cases to directory set by TEST_DUMP_DIR environment variable") - parser.add_argument("--previous_release", dest="previous_release", default=False, action="store_true", - help="Use a previous release as taproot-inactive node") def skip_test_if_missing_module(self): self.skip_if_no_wallet() - if self.options.previous_release: - self.skip_if_no_previous_releases() def set_test_params(self): - self.num_nodes = 2 + self.num_nodes = 1 self.setup_clean_chain = True - # Node 0 has Taproot inactive, Node 1 active. - self.extra_args = [["-par=1"], ["-par=1"]] - if self.options.previous_release: - self.wallet_names = [None, self.default_wallet_name] - else: - self.extra_args[0].append("-vbparams=taproot:1:1") - - def setup_nodes(self): - self.add_nodes(self.num_nodes, self.extra_args, versions=[ - 200100 if self.options.previous_release else None, - None, - ]) - self.start_nodes() - self.import_deterministic_coinbase_privkeys() + self.extra_args = [["-par=1"]] def block_submit(self, node, txs, msg, err_msg, cb_pubkey=None, fees=0, sigops_weight=0, witness=False, accept=False): @@ -1483,11 +1446,10 @@ class TaprootTest(BitcoinTestFramework): for i in range(len(input_utxos)): tx.vin[i].scriptSig = input_data[i][i != fail_input][0] tx.wit.vtxinwit[i].scriptWitness.stack = input_data[i][i != fail_input][1] - taproot_spend_policy = Standard.V23 if node.version is None else Standard.ALL # Submit to mempool to check standardness is_standard_tx = ( fail_input is None # Must be valid to be standard - and (all(utxo.spender.is_standard == Standard.ALL or utxo.spender.is_standard == taproot_spend_policy for utxo in input_utxos)) # All inputs must be standard + and (all(utxo.spender.is_standard for utxo in input_utxos)) # All inputs must be standard and tx.nVersion >= 1 # The tx version must be standard and tx.nVersion <= 2) tx.rehash() @@ -1514,7 +1476,7 @@ class TaprootTest(BitcoinTestFramework): self.log.info("Unit test scenario...") # Deterministically mine coins to OP_TRUE in block 1 - assert self.nodes[1].getblockcount() == 0 + assert_equal(self.nodes[0].getblockcount(), 0) coinbase = CTransaction() coinbase.nVersion = 1 coinbase.vin = [CTxIn(COutPoint(0, 0xffffffff), CScript([OP_1, OP_1]), SEQUENCE_FINAL)] @@ -1523,12 +1485,12 @@ class TaprootTest(BitcoinTestFramework): coinbase.rehash() assert coinbase.hash == "f60c73405d499a956d3162e3483c395526ef78286458a4cb17b125aa92e49b20" # Mine it - block = create_block(hashprev=int(self.nodes[1].getbestblockhash(), 16), coinbase=coinbase) + block = create_block(hashprev=int(self.nodes[0].getbestblockhash(), 16), coinbase=coinbase) block.rehash() block.solve() - self.nodes[1].submitblock(block.serialize().hex()) - assert self.nodes[1].getblockcount() == 1 - self.generate(self.nodes[1], COINBASE_MATURITY) + self.nodes[0].submitblock(block.serialize().hex()) + assert_equal(self.nodes[0].getblockcount(), 1) + self.generate(self.nodes[0], COINBASE_MATURITY) SEED = 317 VALID_LEAF_VERS = list(range(0xc0, 0x100, 2)) + [0x66, 0x7e, 0x80, 0x84, 0x96, 0x98, 0xba, 0xbc, 0xbe] @@ -1617,8 +1579,8 @@ class TaprootTest(BitcoinTestFramework): spend_info[spk]['prevout'] = COutPoint(tx.sha256, i & 1) spend_info[spk]['utxo'] = CTxOut(val, spk) # Mine those transactions - self.init_blockinfo(self.nodes[1]) - self.block_submit(self.nodes[1], txn, "Crediting txn", None, sigops_weight=10, accept=True) + self.init_blockinfo(self.nodes[0]) + self.block_submit(self.nodes[0], txn, "Crediting txn", None, sigops_weight=10, accept=True) # scriptPubKey computation tests = {"version": 1} @@ -1730,53 +1692,21 @@ class TaprootTest(BitcoinTestFramework): keypath_tests.append(tx_test) assert_equal(hashlib.sha256(tx.serialize()).hexdigest(), "24bab662cb55a7f3bae29b559f651674c62bcc1cd442d44715c0133939107b38") # Mine the spending transaction - self.block_submit(self.nodes[1], [tx], "Spending txn", None, sigops_weight=10000, accept=True, witness=True) + self.block_submit(self.nodes[0], [tx], "Spending txn", None, sigops_weight=10000, accept=True, witness=True) if GEN_TEST_VECTORS: print(json.dumps(tests, indent=4, sort_keys=False)) - def run_test(self): self.gen_test_vectors() - # Post-taproot activation tests go first (pre-taproot tests' blocks are invalid post-taproot). self.log.info("Post-activation tests...") - self.test_spenders(self.nodes[1], spenders_taproot_active(), input_counts=[1, 2, 2, 2, 2, 3]) - - # Re-connect nodes in case they have been disconnected - self.disconnect_nodes(0, 1) - self.connect_nodes(0, 1) - - # Transfer value of the largest 500 coins to pre-taproot node. - addr = self.nodes[0].getnewaddress() - - unsp = self.nodes[1].listunspent() - unsp = sorted(unsp, key=lambda i: i['amount'], reverse=True) - unsp = unsp[:500] - - rawtx = self.nodes[1].createrawtransaction( - inputs=[{ - 'txid': i['txid'], - 'vout': i['vout'] - } for i in unsp], - outputs={addr: sum(i['amount'] for i in unsp)} - ) - rawtx = self.nodes[1].signrawtransactionwithwallet(rawtx)['hex'] - - # Mine a block with the transaction - block = create_block(tmpl=self.nodes[1].getblocktemplate(NORMAL_GBT_REQUEST_PARAMS), txlist=[rawtx]) - add_witness_commitment(block) - block.solve() - assert_equal(None, self.nodes[1].submitblock(block.serialize().hex())) - self.sync_blocks() - - # Pre-taproot activation tests. - self.log.info("Pre-activation tests...") + self.test_spenders(self.nodes[0], spenders_taproot_active(), input_counts=[1, 2, 2, 2, 2, 3]) # Run each test twice; once in isolation, and once combined with others. Testing in isolation # means that the standardness is verified in every test (as combined transactions are only standard # when all their inputs are standard). - self.test_spenders(self.nodes[0], spenders_taproot_inactive(), input_counts=[1]) - self.test_spenders(self.nodes[0], spenders_taproot_inactive(), input_counts=[2, 3]) + self.test_spenders(self.nodes[0], spenders_taproot_nonstandard(), input_counts=[1]) + self.test_spenders(self.nodes[0], spenders_taproot_nonstandard(), input_counts=[2, 3]) if __name__ == '__main__': diff --git a/test/functional/feature_unsupported_utxo_db.py b/test/functional/feature_unsupported_utxo_db.py new file mode 100755 index 0000000000..1c8c08d1d8 --- /dev/null +++ b/test/functional/feature_unsupported_utxo_db.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test that unsupported utxo db causes an init error. + +Previous releases are required by this test, see test/README.md. +""" + +import shutil + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + + +class UnsupportedUtxoDbTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 2 + + def skip_test_if_missing_module(self): + self.skip_if_no_previous_releases() + + def setup_network(self): + self.add_nodes( + self.num_nodes, + versions=[ + 140300, # Last release with previous utxo db format + None, # For MiniWallet, without migration code + ], + ) + + def run_test(self): + self.log.info("Create previous version (v0.14.3) utxo db") + self.start_node(0) + block = self.generate(self.nodes[0], 1, sync_fun=self.no_op)[-1] + assert_equal(self.nodes[0].getbestblockhash(), block) + assert_equal(self.nodes[0].gettxoutsetinfo()["total_amount"], 50) + self.stop_nodes() + + self.log.info("Check init error") + legacy_utxos_dir = self.nodes[0].chain_path / "chainstate" + legacy_blocks_dir = self.nodes[0].chain_path / "blocks" + recent_utxos_dir = self.nodes[1].chain_path / "chainstate" + recent_blocks_dir = self.nodes[1].chain_path / "blocks" + shutil.copytree(legacy_utxos_dir, recent_utxos_dir) + shutil.copytree(legacy_blocks_dir, recent_blocks_dir) + self.nodes[1].assert_start_raises_init_error( + expected_msg="Error: Unsupported chainstate database format found. " + "Please restart with -reindex-chainstate. " + "This will rebuild the chainstate database.", + ) + + self.log.info("Drop legacy utxo db") + self.start_node(1, extra_args=["-reindex-chainstate"]) + assert_equal(self.nodes[1].getbestblockhash(), block) + assert_equal(self.nodes[1].gettxoutsetinfo()["total_amount"], 50) + + +if __name__ == "__main__": + UnsupportedUtxoDbTest().main() diff --git a/test/functional/feature_utxo_set_hash.py b/test/functional/feature_utxo_set_hash.py index 75180e62a2..4d486bc6f4 100755 --- a/test/functional/feature_utxo_set_hash.py +++ b/test/functional/feature_utxo_set_hash.py @@ -69,8 +69,8 @@ class UTXOSetHashTest(BitcoinTestFramework): assert_equal(finalized[::-1].hex(), node_muhash) self.log.info("Test deterministic UTXO set hash results") - assert_equal(node.gettxoutsetinfo()['hash_serialized_2'], "3a570529b4c32e77268de1f81b903c75cc2da53c48df0d125c1e697ba7c8c7b7") - assert_equal(node.gettxoutsetinfo("muhash")['muhash'], "a13e0e70eb8acc786549596e3bc154623f1a5a622ba2f70715f6773ec745f435") + assert_equal(node.gettxoutsetinfo()['hash_serialized_2'], "f9aa4fb5ffd10489b9a6994e70ccf1de8a8bfa2d5f201d9857332e9954b0855d") + assert_equal(node.gettxoutsetinfo("muhash")['muhash'], "d1725b2fe3ef43e55aa4907480aea98d406fc9e0bf8f60169e2305f1fbf5961b") def run_test(self): self.test_muhash_implementation() diff --git a/test/functional/feature_versionbits_warning.py b/test/functional/feature_versionbits_warning.py index e83dd7f446..1572463308 100755 --- a/test/functional/feature_versionbits_warning.py +++ b/test/functional/feature_versionbits_warning.py @@ -58,7 +58,8 @@ class VersionBitsWarningTest(BitcoinTestFramework): def versionbits_in_alert_file(self): """Test that the versionbits warning has been written to the alert file.""" - alert_text = open(self.alert_filename, 'r', encoding='utf8').read() + with open(self.alert_filename, 'r', encoding='utf8') as f: + alert_text = f.read() return VB_PATTERN.search(alert_text) is not None def run_test(self): diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index a3d949c6a8..2c158e37e7 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -10,6 +10,7 @@ import http.client from io import BytesIO import json from struct import pack, unpack +import typing import urllib.parse @@ -57,14 +58,21 @@ class RESTTest (BitcoinTestFramework): args.append("-whitelist=noban@127.0.0.1") self.supports_cli = False - def test_rest_request(self, uri, http_method='GET', req_type=ReqType.JSON, body='', status=200, ret_type=RetType.JSON): + def test_rest_request( + self, + uri: str, + http_method: str = 'GET', + req_type: ReqType = ReqType.JSON, + body: str = '', + status: int = 200, + ret_type: RetType = RetType.JSON, + query_params: typing.Dict[str, typing.Any] = None, + ) -> typing.Union[http.client.HTTPResponse, bytes, str, None]: rest_uri = '/rest' + uri - if req_type == ReqType.JSON: - rest_uri += '.json' - elif req_type == ReqType.BIN: - rest_uri += '.bin' - elif req_type == ReqType.HEX: - rest_uri += '.hex' + if req_type in ReqType: + rest_uri += f'.{req_type.name.lower()}' + if query_params: + rest_uri += f'?{urllib.parse.urlencode(query_params)}' conn = http.client.HTTPConnection(self.url.hostname, self.url.port) self.log.debug(f'{http_method} {rest_uri} {body}') @@ -83,6 +91,8 @@ class RESTTest (BitcoinTestFramework): elif ret_type == RetType.JSON: return json.loads(resp.read().decode('utf-8'), parse_float=Decimal) + return None + def run_test(self): self.url = urllib.parse.urlparse(self.nodes[0].url) self.wallet = MiniWallet(self.nodes[0]) @@ -209,16 +219,16 @@ class RESTTest (BitcoinTestFramework): self.generate(self.nodes[0], 1) # generate block to not affect upcoming tests - self.log.info("Test the /block, /blockhashbyheight and /headers URIs") + self.log.info("Test the /block, /blockhashbyheight, /headers, and /blockfilterheaders URIs") bb_hash = self.nodes[0].getbestblockhash() # Check result if block does not exists - assert_equal(self.test_rest_request(f"/headers/1/{UNKNOWN_PARAM}"), []) + assert_equal(self.test_rest_request(f"/headers/{UNKNOWN_PARAM}", query_params={"count": 1}), []) self.test_rest_request(f"/block/{UNKNOWN_PARAM}", status=404, ret_type=RetType.OBJ) # Check result if block is not in the active chain self.nodes[0].invalidateblock(bb_hash) - assert_equal(self.test_rest_request(f'/headers/1/{bb_hash}'), []) + assert_equal(self.test_rest_request(f'/headers/{bb_hash}', query_params={'count': 1}), []) self.test_rest_request(f'/block/{bb_hash}') self.nodes[0].reconsiderblock(bb_hash) @@ -228,7 +238,7 @@ class RESTTest (BitcoinTestFramework): response_bytes = response.read() # Compare with block header - response_header = self.test_rest_request(f"/headers/1/{bb_hash}", req_type=ReqType.BIN, ret_type=RetType.OBJ) + response_header = self.test_rest_request(f"/headers/{bb_hash}", req_type=ReqType.BIN, ret_type=RetType.OBJ, query_params={"count": 1}) assert_equal(int(response_header.getheader('content-length')), BLOCK_HEADER_SIZE) response_header_bytes = response_header.read() assert_equal(response_bytes[:BLOCK_HEADER_SIZE], response_header_bytes) @@ -240,7 +250,7 @@ class RESTTest (BitcoinTestFramework): assert_equal(response_bytes.hex().encode(), response_hex_bytes) # Compare with hex block header - response_header_hex = self.test_rest_request(f"/headers/1/{bb_hash}", req_type=ReqType.HEX, ret_type=RetType.OBJ) + response_header_hex = self.test_rest_request(f"/headers/{bb_hash}", req_type=ReqType.HEX, ret_type=RetType.OBJ, query_params={"count": 1}) assert_greater_than(int(response_header_hex.getheader('content-length')), BLOCK_HEADER_SIZE*2) response_header_hex_bytes = response_header_hex.read(BLOCK_HEADER_SIZE*2) assert_equal(response_bytes[:BLOCK_HEADER_SIZE].hex().encode(), response_header_hex_bytes) @@ -267,7 +277,7 @@ class RESTTest (BitcoinTestFramework): self.test_rest_request("/blockhashbyheight/", ret_type=RetType.OBJ, status=400) # Compare with json block header - json_obj = self.test_rest_request(f"/headers/1/{bb_hash}") + json_obj = self.test_rest_request(f"/headers/{bb_hash}", query_params={"count": 1}) assert_equal(len(json_obj), 1) # ensure that there is one header in the json response assert_equal(json_obj[0]['hash'], bb_hash) # request/response hash should be the same @@ -278,9 +288,9 @@ class RESTTest (BitcoinTestFramework): # See if we can get 5 headers in one response self.generate(self.nodes[1], 5) - json_obj = self.test_rest_request(f"/headers/5/{bb_hash}") + json_obj = self.test_rest_request(f"/headers/{bb_hash}", query_params={"count": 5}) assert_equal(len(json_obj), 5) # now we should have 5 header objects - json_obj = self.test_rest_request(f"/blockfilterheaders/basic/5/{bb_hash}") + json_obj = self.test_rest_request(f"/blockfilterheaders/basic/{bb_hash}", query_params={"count": 5}) first_filter_header = json_obj[0] assert_equal(len(json_obj), 5) # now we should have 5 filter header objects json_obj = self.test_rest_request(f"/blockfilter/basic/{bb_hash}") @@ -290,11 +300,17 @@ class RESTTest (BitcoinTestFramework): assert_equal(first_filter_header, rpc_blockfilter['header']) assert_equal(json_obj['filter'], rpc_blockfilter['filter']) + # Test blockfilterheaders with an invalid hash and filtertype + resp = self.test_rest_request(f"/blockfilterheaders/{INVALID_PARAM}/{bb_hash}", ret_type=RetType.OBJ, status=400) + assert_equal(resp.read().decode('utf-8').rstrip(), f"Unknown filtertype {INVALID_PARAM}") + resp = self.test_rest_request(f"/blockfilterheaders/basic/{INVALID_PARAM}", ret_type=RetType.OBJ, status=400) + assert_equal(resp.read().decode('utf-8').rstrip(), f"Invalid hash: {INVALID_PARAM}") + # Test number parsing for num in ['5a', '-5', '0', '2001', '99999999999999999999999999999999999']: assert_equal( bytes(f'Header count is invalid or out of acceptable range (1-2000): {num}\r\n', 'ascii'), - self.test_rest_request(f"/headers/{num}/{bb_hash}", ret_type=RetType.BYTES, status=400), + self.test_rest_request(f"/headers/{bb_hash}", ret_type=RetType.BYTES, status=400, query_params={"count": num}), ) self.log.info("Test tx inclusion in the /mempool and /block URIs") @@ -316,6 +332,10 @@ class RESTTest (BitcoinTestFramework): # Check that there are our submitted transactions in the TX memory pool json_obj = self.test_rest_request("/mempool/contents") + raw_mempool_verbose = self.nodes[0].getrawmempool(verbose=True) + + assert_equal(json_obj, raw_mempool_verbose) + for i, tx in enumerate(txs): assert tx in json_obj assert_equal(json_obj[tx]['spentby'], txs[i + 1:i + 2]) @@ -351,6 +371,15 @@ class RESTTest (BitcoinTestFramework): json_obj = self.test_rest_request("/chaininfo") assert_equal(json_obj['bestblockhash'], bb_hash) + # Compare with normal RPC getblockchaininfo response + blockchain_info = self.nodes[0].getblockchaininfo() + assert_equal(blockchain_info, json_obj) + + # Test compatibility of deprecated and newer endpoints + self.log.info("Test compatibility of deprecated and newer endpoints") + assert_equal(self.test_rest_request(f"/headers/{bb_hash}", query_params={"count": 1}), self.test_rest_request(f"/headers/1/{bb_hash}")) + assert_equal(self.test_rest_request(f"/blockfilterheaders/basic/{bb_hash}", query_params={"count": 1}), self.test_rest_request(f"/blockfilterheaders/basic/5/{bb_hash}")) + if __name__ == '__main__': RESTTest().main() diff --git a/test/functional/interface_usdt_coinselection.py b/test/functional/interface_usdt_coinselection.py new file mode 100755 index 0000000000..ef32feda99 --- /dev/null +++ b/test/functional/interface_usdt_coinselection.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" Tests the coin_selection:* tracepoint API interface. + See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#context-coin_selection +""" + +# Test will be skipped if we don't have bcc installed +try: + from bcc import BPF, USDT # type: ignore[import] +except ImportError: + pass +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_rpc_error, +) + +coinselection_tracepoints_program = """ +#include <uapi/linux/ptrace.h> + +#define WALLET_NAME_LENGTH 16 +#define ALGO_NAME_LENGTH 16 + +struct event_data +{ + u8 type; + char wallet_name[WALLET_NAME_LENGTH]; + + // selected coins event + char algo[ALGO_NAME_LENGTH]; + s64 target; + s64 waste; + s64 selected_value; + + // create tx event + bool success; + s64 fee; + s32 change_pos; + + // aps create tx event + bool use_aps; +}; + +BPF_QUEUE(coin_selection_events, struct event_data, 1024); + +int trace_selected_coins(struct pt_regs *ctx) { + struct event_data data; + __builtin_memset(&data, 0, sizeof(data)); + data.type = 1; + bpf_usdt_readarg_p(1, ctx, &data.wallet_name, WALLET_NAME_LENGTH); + bpf_usdt_readarg_p(2, ctx, &data.algo, ALGO_NAME_LENGTH); + bpf_usdt_readarg(3, ctx, &data.target); + bpf_usdt_readarg(4, ctx, &data.waste); + bpf_usdt_readarg(5, ctx, &data.selected_value); + coin_selection_events.push(&data, 0); + return 0; +} + +int trace_normal_create_tx(struct pt_regs *ctx) { + struct event_data data; + __builtin_memset(&data, 0, sizeof(data)); + data.type = 2; + bpf_usdt_readarg_p(1, ctx, &data.wallet_name, WALLET_NAME_LENGTH); + bpf_usdt_readarg(2, ctx, &data.success); + bpf_usdt_readarg(3, ctx, &data.fee); + bpf_usdt_readarg(4, ctx, &data.change_pos); + coin_selection_events.push(&data, 0); + return 0; +} + +int trace_attempt_aps(struct pt_regs *ctx) { + struct event_data data; + __builtin_memset(&data, 0, sizeof(data)); + data.type = 3; + bpf_usdt_readarg_p(1, ctx, &data.wallet_name, WALLET_NAME_LENGTH); + coin_selection_events.push(&data, 0); + return 0; +} + +int trace_aps_create_tx(struct pt_regs *ctx) { + struct event_data data; + __builtin_memset(&data, 0, sizeof(data)); + data.type = 4; + bpf_usdt_readarg_p(1, ctx, &data.wallet_name, WALLET_NAME_LENGTH); + bpf_usdt_readarg(2, ctx, &data.use_aps); + bpf_usdt_readarg(3, ctx, &data.success); + bpf_usdt_readarg(4, ctx, &data.fee); + bpf_usdt_readarg(5, ctx, &data.change_pos); + coin_selection_events.push(&data, 0); + return 0; +} +""" + + +class CoinSelectionTracepointTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def skip_test_if_missing_module(self): + self.skip_if_platform_not_linux() + self.skip_if_no_bitcoind_tracepoints() + self.skip_if_no_python_bcc() + self.skip_if_no_bpf_permissions() + self.skip_if_no_wallet() + + def get_tracepoints(self, expected_types): + events = [] + try: + for i in range(0, len(expected_types) + 1): + event = self.bpf["coin_selection_events"].pop() + assert_equal(event.wallet_name.decode(), self.default_wallet_name) + assert_equal(event.type, expected_types[i]) + events.append(event) + else: + # If the loop exits successfully instead of throwing a KeyError, then we have had + # more events than expected. There should be no more than len(expected_types) events. + assert False + except KeyError: + assert_equal(len(events), len(expected_types)) + return events + + + def determine_selection_from_usdt(self, events): + success = None + use_aps = None + algo = None + waste = None + change_pos = None + + is_aps = False + sc_events = [] + for event in events: + if event.type == 1: + if not is_aps: + algo = event.algo.decode() + waste = event.waste + sc_events.append(event) + elif event.type == 2: + success = event.success + if not is_aps: + change_pos = event.change_pos + elif event.type == 3: + is_aps = True + elif event.type == 4: + assert is_aps + if event.use_aps: + use_aps = True + assert_equal(len(sc_events), 2) + algo = sc_events[1].algo.decode() + waste = sc_events[1].waste + change_pos = event.change_pos + return success, use_aps, algo, waste, change_pos + + def run_test(self): + self.log.info("hook into the coin_selection tracepoints") + ctx = USDT(pid=self.nodes[0].process.pid) + ctx.enable_probe(probe="coin_selection:selected_coins", fn_name="trace_selected_coins") + ctx.enable_probe(probe="coin_selection:normal_create_tx_internal", fn_name="trace_normal_create_tx") + ctx.enable_probe(probe="coin_selection:attempting_aps_create_tx", fn_name="trace_attempt_aps") + ctx.enable_probe(probe="coin_selection:aps_create_tx_internal", fn_name="trace_aps_create_tx") + self.bpf = BPF(text=coinselection_tracepoints_program, usdt_contexts=[ctx], debug=0) + + self.log.info("Prepare wallets") + self.generate(self.nodes[0], 101) + wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + + self.log.info("Sending a transaction should result in all tracepoints") + # We should have 5 tracepoints in the order: + # 1. selected_coins (type 1) + # 2. normal_create_tx_internal (type 2) + # 3. attempting_aps_create_tx (type 3) + # 4. selected_coins (type 1) + # 5. aps_create_tx_internal (type 4) + wallet.sendtoaddress(wallet.getnewaddress(), 10) + events = self.get_tracepoints([1, 2, 3, 1, 4]) + success, use_aps, algo, waste, change_pos = self.determine_selection_from_usdt(events) + assert_equal(success, True) + assert_greater_than(change_pos, -1) + + self.log.info("Failing to fund results in 1 tracepoint") + # We should have 1 tracepoints in the order + # 1. normal_create_tx_internal (type 2) + assert_raises_rpc_error(-6, "Insufficient funds", wallet.sendtoaddress, wallet.getnewaddress(), 102 * 50) + events = self.get_tracepoints([2]) + success, use_aps, algo, waste, change_pos = self.determine_selection_from_usdt(events) + assert_equal(success, False) + + self.log.info("Explicitly enabling APS results in 2 tracepoints") + # We should have 2 tracepoints in the order + # 1. selected_coins (type 1) + # 2. normal_create_tx_internal (type 2) + wallet.setwalletflag("avoid_reuse") + wallet.sendtoaddress(address=wallet.getnewaddress(), amount=10, avoid_reuse=True) + events = self.get_tracepoints([1, 2]) + success, use_aps, algo, waste, change_pos = self.determine_selection_from_usdt(events) + assert_equal(success, True) + assert_equal(use_aps, None) + + self.bpf.cleanup() + + +if __name__ == '__main__': + CoinSelectionTracepointTest().main() diff --git a/test/functional/interface_usdt_net.py b/test/functional/interface_usdt_net.py new file mode 100755 index 0000000000..9522cd8c59 --- /dev/null +++ b/test/functional/interface_usdt_net.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" Tests the net:* tracepoint API interface. + See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#context-net +""" + +import ctypes +from io import BytesIO +# Test will be skipped if we don't have bcc installed +try: + from bcc import BPF, USDT # type: ignore[import] +except ImportError: + pass +from test_framework.messages import msg_version +from test_framework.p2p import P2PInterface +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +# Tor v3 addresses are 62 chars + 6 chars for the port (':12345'). +MAX_PEER_ADDR_LENGTH = 68 +MAX_PEER_CONN_TYPE_LENGTH = 20 +MAX_MSG_TYPE_LENGTH = 20 +# We won't process messages larger than 150 byte in this test. For reading +# larger messanges see contrib/tracing/log_raw_p2p_msgs.py +MAX_MSG_DATA_LENGTH = 150 + +net_tracepoints_program = """ +#include <uapi/linux/ptrace.h> + +#define MAX_PEER_ADDR_LENGTH {} +#define MAX_PEER_CONN_TYPE_LENGTH {} +#define MAX_MSG_TYPE_LENGTH {} +#define MAX_MSG_DATA_LENGTH {} +""".format( + MAX_PEER_ADDR_LENGTH, + MAX_PEER_CONN_TYPE_LENGTH, + MAX_MSG_TYPE_LENGTH, + MAX_MSG_DATA_LENGTH +) + """ +#define MIN(a,b) ({ __typeof__ (a) _a = (a); __typeof__ (b) _b = (b); _a < _b ? _a : _b; }) + +struct p2p_message +{ + u64 peer_id; + char peer_addr[MAX_PEER_ADDR_LENGTH]; + char peer_conn_type[MAX_PEER_CONN_TYPE_LENGTH]; + char msg_type[MAX_MSG_TYPE_LENGTH]; + u64 msg_size; + u8 msg[MAX_MSG_DATA_LENGTH]; +}; + +BPF_PERF_OUTPUT(inbound_messages); +int trace_inbound_message(struct pt_regs *ctx) { + struct p2p_message msg = {}; + bpf_usdt_readarg(1, ctx, &msg.peer_id); + bpf_usdt_readarg_p(2, ctx, &msg.peer_addr, MAX_PEER_ADDR_LENGTH); + bpf_usdt_readarg_p(3, ctx, &msg.peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); + bpf_usdt_readarg_p(4, ctx, &msg.msg_type, MAX_MSG_TYPE_LENGTH); + bpf_usdt_readarg(5, ctx, &msg.msg_size); + bpf_usdt_readarg_p(6, ctx, &msg.msg, MIN(msg.msg_size, MAX_MSG_DATA_LENGTH)); + inbound_messages.perf_submit(ctx, &msg, sizeof(msg)); + return 0; +} + +BPF_PERF_OUTPUT(outbound_messages); +int trace_outbound_message(struct pt_regs *ctx) { + struct p2p_message msg = {}; + bpf_usdt_readarg(1, ctx, &msg.peer_id); + bpf_usdt_readarg_p(2, ctx, &msg.peer_addr, MAX_PEER_ADDR_LENGTH); + bpf_usdt_readarg_p(3, ctx, &msg.peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); + bpf_usdt_readarg_p(4, ctx, &msg.msg_type, MAX_MSG_TYPE_LENGTH); + bpf_usdt_readarg(5, ctx, &msg.msg_size); + bpf_usdt_readarg_p(6, ctx, &msg.msg, MIN(msg.msg_size, MAX_MSG_DATA_LENGTH)); + outbound_messages.perf_submit(ctx, &msg, sizeof(msg)); + return 0; +}; +""" + + +class NetTracepointTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_platform_not_linux() + self.skip_if_no_bitcoind_tracepoints() + self.skip_if_no_python_bcc() + self.skip_if_no_bpf_permissions() + + def run_test(self): + # Tests the net:inbound_message and net:outbound_message tracepoints + # See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#context-net + + class P2PMessage(ctypes.Structure): + _fields_ = [ + ("peer_id", ctypes.c_uint64), + ("peer_addr", ctypes.c_char * MAX_PEER_ADDR_LENGTH), + ("peer_conn_type", ctypes.c_char * MAX_PEER_CONN_TYPE_LENGTH), + ("msg_type", ctypes.c_char * MAX_MSG_TYPE_LENGTH), + ("msg_size", ctypes.c_uint64), + ("msg", ctypes.c_ubyte * MAX_MSG_DATA_LENGTH), + ] + + def __repr__(self): + return f"P2PMessage(peer={self.peer_id}, addr={self.peer_addr.decode('utf-8')}, conn_type={self.peer_conn_type.decode('utf-8')}, msg_type={self.msg_type.decode('utf-8')}, msg_size={self.msg_size})" + + self.log.info( + "hook into the net:inbound_message and net:outbound_message tracepoints") + ctx = USDT(path=str(self.options.bitcoind)) + ctx.enable_probe(probe="net:inbound_message", + fn_name="trace_inbound_message") + ctx.enable_probe(probe="net:outbound_message", + fn_name="trace_outbound_message") + bpf = BPF(text=net_tracepoints_program, usdt_contexts=[ctx], debug=0) + + # The handle_* function is a ctypes callback function called from C. When + # we assert in the handle_* function, the AssertError doesn't propagate + # back to Python. The exception is ignored. We manually count and assert + # that the handle_* functions succeeded. + EXPECTED_INOUTBOUND_VERSION_MSG = 1 + checked_inbound_version_msg = 0 + checked_outbound_version_msg = 0 + + def check_p2p_message(event, inbound): + nonlocal checked_inbound_version_msg, checked_outbound_version_msg + if event.msg_type.decode("utf-8") == "version": + self.log.info( + f"check_p2p_message(): {'inbound' if inbound else 'outbound'} {event}") + peer = self.nodes[0].getpeerinfo()[0] + msg = msg_version() + msg.deserialize(BytesIO(bytes(event.msg[:event.msg_size]))) + assert_equal(peer["id"], event.peer_id, peer["id"]) + assert_equal(peer["addr"], event.peer_addr.decode("utf-8")) + assert_equal(peer["connection_type"], + event.peer_conn_type.decode("utf-8")) + if inbound: + checked_inbound_version_msg += 1 + else: + checked_outbound_version_msg += 1 + + def handle_inbound(_, data, __): + event = ctypes.cast(data, ctypes.POINTER(P2PMessage)).contents + check_p2p_message(event, True) + + def handle_outbound(_, data, __): + event = ctypes.cast(data, ctypes.POINTER(P2PMessage)).contents + check_p2p_message(event, False) + + bpf["inbound_messages"].open_perf_buffer(handle_inbound) + bpf["outbound_messages"].open_perf_buffer(handle_outbound) + + self.log.info("connect a P2P test node to our bitcoind node") + test_node = P2PInterface() + self.nodes[0].add_p2p_connection(test_node) + bpf.perf_buffer_poll(timeout=200) + + self.log.info( + "check that we got both an inbound and outbound version message") + assert_equal(EXPECTED_INOUTBOUND_VERSION_MSG, + checked_inbound_version_msg) + assert_equal(EXPECTED_INOUTBOUND_VERSION_MSG, + checked_outbound_version_msg) + + bpf.cleanup() + + +if __name__ == '__main__': + NetTracepointTest().main() diff --git a/test/functional/interface_usdt_utxocache.py b/test/functional/interface_usdt_utxocache.py new file mode 100755 index 0000000000..0c7f351e66 --- /dev/null +++ b/test/functional/interface_usdt_utxocache.py @@ -0,0 +1,407 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" Tests the utxocache:* tracepoint API interface. + See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#context-utxocache +""" + +import ctypes +# Test will be skipped if we don't have bcc installed +try: + from bcc import BPF, USDT # type: ignore[import] +except ImportError: + pass +from test_framework.messages import COIN +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal +from test_framework.wallet import MiniWallet + +utxocache_changes_program = """ +#include <uapi/linux/ptrace.h> + +typedef signed long long i64; + +struct utxocache_change +{ + char txid[32]; + u32 index; + u32 height; + i64 value; + bool is_coinbase; +}; + +BPF_PERF_OUTPUT(utxocache_add); +int trace_utxocache_add(struct pt_regs *ctx) { + struct utxocache_change add = {}; + bpf_usdt_readarg_p(1, ctx, &add.txid, 32); + bpf_usdt_readarg(2, ctx, &add.index); + bpf_usdt_readarg(3, ctx, &add.height); + bpf_usdt_readarg(4, ctx, &add.value); + bpf_usdt_readarg(5, ctx, &add.is_coinbase); + utxocache_add.perf_submit(ctx, &add, sizeof(add)); + return 0; +} + +BPF_PERF_OUTPUT(utxocache_spent); +int trace_utxocache_spent(struct pt_regs *ctx) { + struct utxocache_change spent = {}; + bpf_usdt_readarg_p(1, ctx, &spent.txid, 32); + bpf_usdt_readarg(2, ctx, &spent.index); + bpf_usdt_readarg(3, ctx, &spent.height); + bpf_usdt_readarg(4, ctx, &spent.value); + bpf_usdt_readarg(5, ctx, &spent.is_coinbase); + utxocache_spent.perf_submit(ctx, &spent, sizeof(spent)); + return 0; +} + +BPF_PERF_OUTPUT(utxocache_uncache); +int trace_utxocache_uncache(struct pt_regs *ctx) { + struct utxocache_change uncache = {}; + bpf_usdt_readarg_p(1, ctx, &uncache.txid, 32); + bpf_usdt_readarg(2, ctx, &uncache.index); + bpf_usdt_readarg(3, ctx, &uncache.height); + bpf_usdt_readarg(4, ctx, &uncache.value); + bpf_usdt_readarg(5, ctx, &uncache.is_coinbase); + utxocache_uncache.perf_submit(ctx, &uncache, sizeof(uncache)); + return 0; +} +""" + +utxocache_flushes_program = """ +#include <uapi/linux/ptrace.h> + +typedef signed long long i64; + +struct utxocache_flush +{ + i64 duration; + u32 mode; + u64 size; + u64 memory; + bool for_prune; +}; + +BPF_PERF_OUTPUT(utxocache_flush); +int trace_utxocache_flush(struct pt_regs *ctx) { + struct utxocache_flush flush = {}; + bpf_usdt_readarg(1, ctx, &flush.duration); + bpf_usdt_readarg(2, ctx, &flush.mode); + bpf_usdt_readarg(3, ctx, &flush.size); + bpf_usdt_readarg(4, ctx, &flush.memory); + bpf_usdt_readarg(5, ctx, &flush.for_prune); + utxocache_flush.perf_submit(ctx, &flush, sizeof(flush)); + return 0; +} +""" + +FLUSHMODE_NAME = { + 0: "NONE", + 1: "IF_NEEDED", + 2: "PERIODIC", + 3: "ALWAYS", +} + + +class UTXOCacheChange(ctypes.Structure): + _fields_ = [ + ("txid", ctypes.c_ubyte * 32), + ("index", ctypes.c_uint32), + ("height", ctypes.c_uint32), + ("value", ctypes.c_uint64), + ("is_coinbase", ctypes.c_bool), + ] + + def __repr__(self): + return f"UTXOCacheChange(outpoint={bytes(self.txid[::-1]).hex()}:{self.index}, height={self.height}, value={self.value}sat, is_coinbase={self.is_coinbase})" + + +class UTXOCacheFlush(ctypes.Structure): + _fields_ = [ + ("duration", ctypes.c_int64), + ("mode", ctypes.c_uint32), + ("size", ctypes.c_uint64), + ("memory", ctypes.c_uint64), + ("for_prune", ctypes.c_bool), + ] + + def __repr__(self): + return f"UTXOCacheFlush(duration={self.duration}, mode={FLUSHMODE_NAME[self.mode]}, size={self.size}, memory={self.memory}, for_prune={self.for_prune})" + + +class UTXOCacheTracepointTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = False + self.num_nodes = 1 + self.extra_args = [["-txindex"]] + + def skip_test_if_missing_module(self): + self.skip_if_platform_not_linux() + self.skip_if_no_bitcoind_tracepoints() + self.skip_if_no_python_bcc() + self.skip_if_no_bpf_permissions() + + def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + self.generate(self.wallet, 101) + + self.test_uncache() + self.test_add_spent() + self.test_flush() + + def test_uncache(self): + """ Tests the utxocache:uncache tracepoint API. + https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#tracepoint-utxocacheuncache + """ + # To trigger an UTXO uncache from the cache, we create an invalid transaction + # spending a not-cached, but existing UTXO. During transaction validation, this + # the UTXO is added to the utxo cache, but as the transaction is invalid, it's + # uncached again. + self.log.info("testing the utxocache:uncache tracepoint API") + + # Retrieve the txid for the UTXO created in the first block. This UTXO is not + # in our UTXO cache. + EARLY_BLOCK_HEIGHT = 1 + block_1_hash = self.nodes[0].getblockhash(EARLY_BLOCK_HEIGHT) + block_1 = self.nodes[0].getblock(block_1_hash) + block_1_coinbase_txid = block_1["tx"][0] + + # Create a transaction and invalidate it by changing the txid of the previous + # output to the coinbase txid of the block at height 1. + invalid_tx = self.wallet.create_self_transfer( + from_node=self.nodes[0])["tx"] + invalid_tx.vin[0].prevout.hash = int(block_1_coinbase_txid, 16) + + self.log.info("hooking into the utxocache:uncache tracepoint") + ctx = USDT(path=str(self.options.bitcoind)) + ctx.enable_probe(probe="utxocache:uncache", + fn_name="trace_utxocache_uncache") + bpf = BPF(text=utxocache_changes_program, usdt_contexts=[ctx], debug=0) + + # The handle_* function is a ctypes callback function called from C. When + # we assert in the handle_* function, the AssertError doesn't propagate + # back to Python. The exception is ignored. We manually count and assert + # that the handle_* functions succeeded. + EXPECTED_HANDLE_UNCACHE_SUCCESS = 1 + handle_uncache_succeeds = 0 + + def handle_utxocache_uncache(_, data, __): + nonlocal handle_uncache_succeeds + event = ctypes.cast(data, ctypes.POINTER(UTXOCacheChange)).contents + self.log.info(f"handle_utxocache_uncache(): {event}") + assert_equal(block_1_coinbase_txid, bytes(event.txid[::-1]).hex()) + assert_equal(0, event.index) # prevout index + assert_equal(EARLY_BLOCK_HEIGHT, event.height) + assert_equal(50 * COIN, event.value) + assert_equal(True, event.is_coinbase) + + handle_uncache_succeeds += 1 + + bpf["utxocache_uncache"].open_perf_buffer(handle_utxocache_uncache) + + self.log.info( + "testmempoolaccept the invalid transaction to trigger an UTXO-cache uncache") + result = self.nodes[0].testmempoolaccept( + [invalid_tx.serialize().hex()])[0] + assert_equal(result["allowed"], False) + + bpf.perf_buffer_poll(timeout=100) + bpf.cleanup() + + self.log.info( + f"check that we successfully traced {EXPECTED_HANDLE_UNCACHE_SUCCESS} uncaches") + assert_equal(EXPECTED_HANDLE_UNCACHE_SUCCESS, handle_uncache_succeeds) + + def test_add_spent(self): + """ Tests the utxocache:add utxocache:spent tracepoint API + See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#tracepoint-utxocacheadd + and https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#tracepoint-utxocachespent + """ + + self.log.info( + "test the utxocache:add and utxocache:spent tracepoint API") + + self.log.info("create an unconfirmed transaction") + self.wallet.send_self_transfer(from_node=self.nodes[0]) + + # We mine a block to trace changes (add/spent) to the active in-memory cache + # of the UTXO set (see CoinsTip() of CCoinsViewCache). However, in some cases + # temporary clones of the active cache are made. For example, during mining with + # the generate RPC call, the block is first tested in TestBlockValidity(). There, + # a clone of the active cache is modified during a test ConnectBlock() call. + # These are implementation details we don't want to test here. Thus, after + # mining, we invalidate the block, start the tracing, and then trace the cache + # changes to the active utxo cache. + self.log.info("mine and invalidate a block that is later reconsidered") + block_hash = self.generate(self.wallet, 1)[0] + self.nodes[0].invalidateblock(block_hash) + + self.log.info( + "hook into the utxocache:add and utxocache:spent tracepoints") + ctx = USDT(path=str(self.options.bitcoind)) + ctx.enable_probe(probe="utxocache:add", fn_name="trace_utxocache_add") + ctx.enable_probe(probe="utxocache:spent", + fn_name="trace_utxocache_spent") + bpf = BPF(text=utxocache_changes_program, usdt_contexts=[ctx], debug=0) + + # The handle_* function is a ctypes callback function called from C. When + # we assert in the handle_* function, the AssertError doesn't propagate + # back to Python. The exception is ignored. We manually count and assert + # that the handle_* functions succeeded. + EXPECTED_HANDLE_ADD_SUCCESS = 2 + EXPECTED_HANDLE_SPENT_SUCCESS = 1 + handle_add_succeeds = 0 + handle_spent_succeeds = 0 + + expected_utxocache_spents = [] + expected_utxocache_adds = [] + + def handle_utxocache_add(_, data, __): + nonlocal handle_add_succeeds + event = ctypes.cast(data, ctypes.POINTER(UTXOCacheChange)).contents + self.log.info(f"handle_utxocache_add(): {event}") + add = expected_utxocache_adds.pop(0) + assert_equal(add["txid"], bytes(event.txid[::-1]).hex()) + assert_equal(add["index"], event.index) + assert_equal(add["height"], event.height) + assert_equal(add["value"], event.value) + assert_equal(add["is_coinbase"], event.is_coinbase) + handle_add_succeeds += 1 + + def handle_utxocache_spent(_, data, __): + nonlocal handle_spent_succeeds + event = ctypes.cast(data, ctypes.POINTER(UTXOCacheChange)).contents + self.log.info(f"handle_utxocache_spent(): {event}") + spent = expected_utxocache_spents.pop(0) + assert_equal(spent["txid"], bytes(event.txid[::-1]).hex()) + assert_equal(spent["index"], event.index) + assert_equal(spent["height"], event.height) + assert_equal(spent["value"], event.value) + assert_equal(spent["is_coinbase"], event.is_coinbase) + handle_spent_succeeds += 1 + + bpf["utxocache_add"].open_perf_buffer(handle_utxocache_add) + bpf["utxocache_spent"].open_perf_buffer(handle_utxocache_spent) + + # We trigger a block re-connection. This causes changes (add/spent) + # to the UTXO-cache which in turn triggers the tracepoints. + self.log.info("reconsider the previously invalidated block") + self.nodes[0].reconsiderblock(block_hash) + + block = self.nodes[0].getblock(block_hash, 2) + for (block_index, tx) in enumerate(block["tx"]): + for vin in tx["vin"]: + if "coinbase" not in vin: + prevout_tx = self.nodes[0].getrawtransaction( + vin["txid"], True) + prevout_tx_block = self.nodes[0].getblockheader( + prevout_tx["blockhash"]) + spends_coinbase = "coinbase" in prevout_tx["vin"][0] + expected_utxocache_spents.append({ + "txid": vin["txid"], + "index": vin["vout"], + "height": prevout_tx_block["height"], + "value": int(prevout_tx["vout"][vin["vout"]]["value"] * COIN), + "is_coinbase": spends_coinbase, + }) + for (i, vout) in enumerate(tx["vout"]): + if vout["scriptPubKey"]["type"] != "nulldata": + expected_utxocache_adds.append({ + "txid": tx["txid"], + "index": i, + "height": block["height"], + "value": int(vout["value"] * COIN), + "is_coinbase": block_index == 0, + }) + + assert_equal(EXPECTED_HANDLE_ADD_SUCCESS, len(expected_utxocache_adds)) + assert_equal(EXPECTED_HANDLE_SPENT_SUCCESS, + len(expected_utxocache_spents)) + + bpf.perf_buffer_poll(timeout=200) + bpf.cleanup() + + self.log.info( + f"check that we successfully traced {EXPECTED_HANDLE_ADD_SUCCESS} adds and {EXPECTED_HANDLE_SPENT_SUCCESS} spent") + assert_equal(0, len(expected_utxocache_adds)) + assert_equal(0, len(expected_utxocache_spents)) + assert_equal(EXPECTED_HANDLE_ADD_SUCCESS, handle_add_succeeds) + assert_equal(EXPECTED_HANDLE_SPENT_SUCCESS, handle_spent_succeeds) + + def test_flush(self): + """ Tests the utxocache:flush tracepoint API. + See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#tracepoint-utxocacheflush""" + + self.log.info("test the utxocache:flush tracepoint API") + self.log.info("hook into the utxocache:flush tracepoint") + ctx = USDT(path=str(self.options.bitcoind)) + ctx.enable_probe(probe="utxocache:flush", + fn_name="trace_utxocache_flush") + bpf = BPF(text=utxocache_flushes_program, usdt_contexts=[ctx], debug=0) + + # The handle_* function is a ctypes callback function called from C. When + # we assert in the handle_* function, the AssertError doesn't propagate + # back to Python. The exception is ignored. We manually count and assert + # that the handle_* functions succeeded. + EXPECTED_HANDLE_FLUSH_SUCCESS = 3 + handle_flush_succeeds = 0 + possible_cache_sizes = set() + expected_flushes = [] + + def handle_utxocache_flush(_, data, __): + nonlocal handle_flush_succeeds + event = ctypes.cast(data, ctypes.POINTER(UTXOCacheFlush)).contents + self.log.info(f"handle_utxocache_flush(): {event}") + expected = expected_flushes.pop(0) + assert_equal(expected["mode"], FLUSHMODE_NAME[event.mode]) + possible_cache_sizes.remove(event.size) # fails if size not in set + # sanity checks only + assert(event.memory > 0) + assert(event.duration > 0) + handle_flush_succeeds += 1 + + bpf["utxocache_flush"].open_perf_buffer(handle_utxocache_flush) + + self.log.info("stop the node to flush the UTXO cache") + UTXOS_IN_CACHE = 104 # might need to be changed if the eariler tests are modified + # A node shutdown causes two flushes. One that flushes UTXOS_IN_CACHE + # UTXOs and one that flushes 0 UTXOs. Normally the 0-UTXO-flush is the + # second flush, however it can happen that the order changes. + possible_cache_sizes = {UTXOS_IN_CACHE, 0} + flush_for_shutdown = {"mode": "ALWAYS", "for_prune": False} + expected_flushes.extend([flush_for_shutdown, flush_for_shutdown]) + self.stop_node(0) + + bpf.perf_buffer_poll(timeout=200) + + self.log.info("check that we don't expect additional flushes") + assert_equal(0, len(expected_flushes)) + assert_equal(0, len(possible_cache_sizes)) + + self.log.info("restart the node with -prune") + self.start_node(0, ["-fastprune=1", "-prune=1"]) + + BLOCKS_TO_MINE = 350 + self.log.info(f"mine {BLOCKS_TO_MINE} blocks to be able to prune") + self.generate(self.wallet, BLOCKS_TO_MINE) + # we added BLOCKS_TO_MINE coinbase UTXOs to the cache + possible_cache_sizes = {BLOCKS_TO_MINE} + expected_flushes.append( + {"mode": "NONE", "for_prune": True, "size_fn": lambda x: x == BLOCKS_TO_MINE}) + + self.log.info(f"prune blockchain to trigger a flush for pruning") + self.nodes[0].pruneblockchain(315) + + bpf.perf_buffer_poll(timeout=500) + bpf.cleanup() + + self.log.info( + f"check that we don't expect additional flushes and that the handle_* function succeeded") + assert_equal(0, len(expected_flushes)) + assert_equal(0, len(possible_cache_sizes)) + assert_equal(EXPECTED_HANDLE_FLUSH_SUCCESS, handle_flush_succeeds) + + +if __name__ == '__main__': + UTXOCacheTracepointTest().main() diff --git a/test/functional/interface_usdt_validation.py b/test/functional/interface_usdt_validation.py new file mode 100755 index 0000000000..d11809273b --- /dev/null +++ b/test/functional/interface_usdt_validation.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" Tests the validation:* tracepoint API interface. + See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#context-validation +""" + +import ctypes + +# Test will be skipped if we don't have bcc installed +try: + from bcc import BPF, USDT # type: ignore[import] +except ImportError: + pass + +from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + + +validation_blockconnected_program = """ +#include <uapi/linux/ptrace.h> + +typedef signed long long i64; + +struct connected_block +{ + char hash[32]; + int height; + i64 transactions; + int inputs; + i64 sigops; + u64 duration; +}; + +BPF_PERF_OUTPUT(block_connected); +int trace_block_connected(struct pt_regs *ctx) { + struct connected_block block = {}; + bpf_usdt_readarg_p(1, ctx, &block.hash, 32); + bpf_usdt_readarg(2, ctx, &block.height); + bpf_usdt_readarg(3, ctx, &block.transactions); + bpf_usdt_readarg(4, ctx, &block.inputs); + bpf_usdt_readarg(5, ctx, &block.sigops); + bpf_usdt_readarg(6, ctx, &block.duration); + block_connected.perf_submit(ctx, &block, sizeof(block)); + return 0; +} +""" + + +class ValidationTracepointTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_platform_not_linux() + self.skip_if_no_bitcoind_tracepoints() + self.skip_if_no_python_bcc() + self.skip_if_no_bpf_permissions() + + def run_test(self): + # Tests the validation:block_connected tracepoint by generating blocks + # and comparing the values passed in the tracepoint arguments with the + # blocks. + # See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#tracepoint-validationblock_connected + + class Block(ctypes.Structure): + _fields_ = [ + ("hash", ctypes.c_ubyte * 32), + ("height", ctypes.c_int), + ("transactions", ctypes.c_int64), + ("inputs", ctypes.c_int), + ("sigops", ctypes.c_int64), + ("duration", ctypes.c_uint64), + ] + + def __repr__(self): + return "ConnectedBlock(hash=%s height=%d, transactions=%d, inputs=%d, sigops=%d, duration=%d)" % ( + bytes(self.hash[::-1]).hex(), + self.height, + self.transactions, + self.inputs, + self.sigops, + self.duration) + + # The handle_* function is a ctypes callback function called from C. When + # we assert in the handle_* function, the AssertError doesn't propagate + # back to Python. The exception is ignored. We manually count and assert + # that the handle_* functions succeeded. + BLOCKS_EXPECTED = 2 + blocks_checked = 0 + expected_blocks = list() + + self.log.info("hook into the validation:block_connected tracepoint") + ctx = USDT(path=str(self.options.bitcoind)) + ctx.enable_probe(probe="validation:block_connected", + fn_name="trace_block_connected") + bpf = BPF(text=validation_blockconnected_program, + usdt_contexts=[ctx], debug=0) + + def handle_blockconnected(_, data, __): + nonlocal expected_blocks, blocks_checked + event = ctypes.cast(data, ctypes.POINTER(Block)).contents + self.log.info(f"handle_blockconnected(): {event}") + block = expected_blocks.pop(0) + assert_equal(block["hash"], bytes(event.hash[::-1]).hex()) + assert_equal(block["height"], event.height) + assert_equal(len(block["tx"]), event.transactions) + assert_equal(len([tx["vin"] for tx in block["tx"]]), event.inputs) + assert_equal(0, event.sigops) # no sigops in coinbase tx + # only plausibility checks + assert(event.duration > 0) + + blocks_checked += 1 + + bpf["block_connected"].open_perf_buffer( + handle_blockconnected) + + self.log.info(f"mine {BLOCKS_EXPECTED} blocks") + block_hashes = self.generatetoaddress( + self.nodes[0], BLOCKS_EXPECTED, ADDRESS_BCRT1_UNSPENDABLE) + for block_hash in block_hashes: + expected_blocks.append(self.nodes[0].getblock(block_hash, 2)) + + bpf.perf_buffer_poll(timeout=200) + bpf.cleanup() + + self.log.info(f"check that we traced {BLOCKS_EXPECTED} blocks") + assert_equal(BLOCKS_EXPECTED, blocks_checked) + assert_equal(0, len(expected_blocks)) + + +if __name__ == '__main__': + ValidationTracepointTest().main() diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 1ee12c0040..7d8d10589b 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -23,6 +23,9 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, ) +from test_framework.wallet import ( + MiniWallet, +) from test_framework.netutil import test_ipv6_local from io import BytesIO from time import sleep @@ -100,8 +103,6 @@ class ZMQTestSetupBlock: class ZMQTest (BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - if self.is_wallet_compiled(): - self.requires_wallet = True # This test isn't testing txn relay/timing, so set whitelist on the # peers for instant txn relay. This speeds up the test run time 2-3x. self.extra_args = [["-whitelist=noban@127.0.0.1"]] * self.num_nodes @@ -111,6 +112,7 @@ class ZMQTest (BitcoinTestFramework): self.skip_if_no_bitcoind_zmq() def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) self.ctx = zmq.Context() try: self.test_basic() @@ -211,25 +213,25 @@ class ZMQTest (BitcoinTestFramework): assert_equal([txid.hex()], self.nodes[1].getblock(hash)["tx"]) - if self.is_wallet_compiled(): - self.log.info("Wait for tx from second node") - payment_txid = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1.0) - self.sync_all() - - # Should receive the broadcasted txid. - txid = hashtx.receive() - assert_equal(payment_txid, txid.hex()) + self.wallet.rescan_utxos() + self.log.info("Wait for tx from second node") + payment_tx = self.wallet.send_self_transfer(from_node=self.nodes[1]) + payment_txid = payment_tx['txid'] + self.sync_all() + # Should receive the broadcasted txid. + txid = hashtx.receive() + assert_equal(payment_txid, txid.hex()) - # Should receive the broadcasted raw transaction. - hex = rawtx.receive() - assert_equal(payment_txid, hash256_reversed(hex).hex()) + # Should receive the broadcasted raw transaction. + hex = rawtx.receive() + assert_equal(payment_tx['wtxid'], hash256_reversed(hex).hex()) - # Mining the block with this tx should result in second notification - # after coinbase tx notification - self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) - hashtx.receive() - txid = hashtx.receive() - assert_equal(payment_txid, txid.hex()) + # Mining the block with this tx should result in second notification + # after coinbase tx notification + self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) + hashtx.receive() + txid = hashtx.receive() + assert_equal(payment_txid, txid.hex()) self.log.info("Test the getzmqnotifications RPC") @@ -243,9 +245,6 @@ class ZMQTest (BitcoinTestFramework): assert_equal(self.nodes[1].getzmqnotifications(), []) def test_reorg(self): - if not self.is_wallet_compiled(): - self.log.info("Skipping reorg test because wallet is disabled") - return address = 'tcp://127.0.0.1:28333' @@ -256,7 +255,7 @@ class ZMQTest (BitcoinTestFramework): self.disconnect_nodes(0, 1) # Generate 1 block in nodes[0] with 1 mempool tx and receive all notifications - payment_txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1.0) + payment_txid = self.wallet.send_self_transfer(from_node=self.nodes[0])['txid'] disconnect_block = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE, sync_fun=self.no_op)[0] disconnect_cb = self.nodes[0].getblock(disconnect_block)["tx"][0] assert_equal(self.nodes[0].getbestblockhash(), hashblock.receive().hex()) @@ -325,126 +324,124 @@ class ZMQTest (BitcoinTestFramework): assert_equal((self.nodes[1].getblockhash(block_count-1), "C", None), seq.receive_sequence()) assert_equal((self.nodes[1].getblockhash(block_count), "C", None), seq.receive_sequence()) - # Rest of test requires wallet functionality - if self.is_wallet_compiled(): - self.log.info("Wait for tx from second node") - payment_txid = self.nodes[1].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=5.0, replaceable=True) - self.sync_all() - self.log.info("Testing sequence notifications with mempool sequence values") - - # Should receive the broadcasted txid. - assert_equal((payment_txid, "A", seq_num), seq.receive_sequence()) - seq_num += 1 - - self.log.info("Testing RBF notification") - # Replace it to test eviction/addition notification - rbf_info = self.nodes[1].bumpfee(payment_txid) - self.sync_all() - assert_equal((payment_txid, "R", seq_num), seq.receive_sequence()) - seq_num += 1 - assert_equal((rbf_info["txid"], "A", seq_num), seq.receive_sequence()) - seq_num += 1 - - # Doesn't get published when mined, make a block and tx to "flush" the possibility - # though the mempool sequence number does go up by the number of transactions - # removed from the mempool by the block mining it. - mempool_size = len(self.nodes[0].getrawmempool()) - c_block = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE)[0] - # Make sure the number of mined transactions matches the number of txs out of mempool - mempool_size_delta = mempool_size - len(self.nodes[0].getrawmempool()) - assert_equal(len(self.nodes[0].getblock(c_block)["tx"])-1, mempool_size_delta) - seq_num += mempool_size_delta - payment_txid_2 = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1.0) - self.sync_all() - assert_equal((c_block, "C", None), seq.receive_sequence()) - assert_equal((payment_txid_2, "A", seq_num), seq.receive_sequence()) - seq_num += 1 - - # Spot check getrawmempool results that they only show up when asked for - assert type(self.nodes[0].getrawmempool()) is list - assert type(self.nodes[0].getrawmempool(mempool_sequence=False)) is list - assert "mempool_sequence" not in self.nodes[0].getrawmempool(verbose=True) - assert_raises_rpc_error(-8, "Verbose results cannot contain mempool sequence values.", self.nodes[0].getrawmempool, True, True) - assert_equal(self.nodes[0].getrawmempool(mempool_sequence=True)["mempool_sequence"], seq_num) - - self.log.info("Testing reorg notifications") - # Manually invalidate the last block to test mempool re-entry - # N.B. This part could be made more lenient in exact ordering - # since it greatly depends on inner-workings of blocks/mempool - # during "deep" re-orgs. Probably should "re-construct" - # blockchain/mempool state from notifications instead. - block_count = self.nodes[0].getblockcount() - best_hash = self.nodes[0].getbestblockhash() - self.nodes[0].invalidateblock(best_hash) - sleep(2) # Bit of room to make sure transaction things happened - - # Make sure getrawmempool mempool_sequence results aren't "queued" but immediately reflective - # of the time they were gathered. - assert self.nodes[0].getrawmempool(mempool_sequence=True)["mempool_sequence"] > seq_num - - assert_equal((best_hash, "D", None), seq.receive_sequence()) - assert_equal((rbf_info["txid"], "A", seq_num), seq.receive_sequence()) - seq_num += 1 - - # Other things may happen but aren't wallet-deterministic so we don't test for them currently - self.nodes[0].reconsiderblock(best_hash) - self.generatetoaddress(self.nodes[1], 1, ADDRESS_BCRT1_UNSPENDABLE) - - self.log.info("Evict mempool transaction by block conflict") - orig_txid = self.nodes[0].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=1.0, replaceable=True) - - # More to be simply mined - more_tx = [] - for _ in range(5): - more_tx.append(self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 0.1)) - - raw_tx = self.nodes[0].getrawtransaction(orig_txid) - bump_info = self.nodes[0].bumpfee(orig_txid) - # Mine the pre-bump tx - txs_to_add = [raw_tx] + [self.nodes[0].getrawtransaction(txid) for txid in more_tx] - block = create_block(int(self.nodes[0].getbestblockhash(), 16), create_coinbase(self.nodes[0].getblockcount()+1), txlist=txs_to_add) - add_witness_commitment(block) - block.solve() - assert_equal(self.nodes[0].submitblock(block.serialize().hex()), None) - tip = self.nodes[0].getbestblockhash() - assert_equal(int(tip, 16), block.sha256) - orig_txid_2 = self.nodes[0].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=1.0, replaceable=True) - - # Flush old notifications until evicted tx original entry + self.log.info("Wait for tx from second node") + payment_tx = self.wallet.send_self_transfer(from_node=self.nodes[1]) + payment_txid = payment_tx['txid'] + self.sync_all() + self.log.info("Testing sequence notifications with mempool sequence values") + + # Should receive the broadcasted txid. + assert_equal((payment_txid, "A", seq_num), seq.receive_sequence()) + seq_num += 1 + + self.log.info("Testing RBF notification") + # Replace it to test eviction/addition notification + payment_tx['tx'].vout[0].nValue -= 1000 + rbf_txid = self.nodes[1].sendrawtransaction(payment_tx['tx'].serialize().hex()) + self.sync_all() + assert_equal((payment_txid, "R", seq_num), seq.receive_sequence()) + seq_num += 1 + assert_equal((rbf_txid, "A", seq_num), seq.receive_sequence()) + seq_num += 1 + + # Doesn't get published when mined, make a block and tx to "flush" the possibility + # though the mempool sequence number does go up by the number of transactions + # removed from the mempool by the block mining it. + mempool_size = len(self.nodes[0].getrawmempool()) + c_block = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE)[0] + # Make sure the number of mined transactions matches the number of txs out of mempool + mempool_size_delta = mempool_size - len(self.nodes[0].getrawmempool()) + assert_equal(len(self.nodes[0].getblock(c_block)["tx"])-1, mempool_size_delta) + seq_num += mempool_size_delta + payment_txid_2 = self.wallet.send_self_transfer(from_node=self.nodes[1])['txid'] + self.sync_all() + assert_equal((c_block, "C", None), seq.receive_sequence()) + assert_equal((payment_txid_2, "A", seq_num), seq.receive_sequence()) + seq_num += 1 + + # Spot check getrawmempool results that they only show up when asked for + assert type(self.nodes[0].getrawmempool()) is list + assert type(self.nodes[0].getrawmempool(mempool_sequence=False)) is list + assert "mempool_sequence" not in self.nodes[0].getrawmempool(verbose=True) + assert_raises_rpc_error(-8, "Verbose results cannot contain mempool sequence values.", self.nodes[0].getrawmempool, True, True) + assert_equal(self.nodes[0].getrawmempool(mempool_sequence=True)["mempool_sequence"], seq_num) + + self.log.info("Testing reorg notifications") + # Manually invalidate the last block to test mempool re-entry + # N.B. This part could be made more lenient in exact ordering + # since it greatly depends on inner-workings of blocks/mempool + # during "deep" re-orgs. Probably should "re-construct" + # blockchain/mempool state from notifications instead. + block_count = self.nodes[0].getblockcount() + best_hash = self.nodes[0].getbestblockhash() + self.nodes[0].invalidateblock(best_hash) + sleep(2) # Bit of room to make sure transaction things happened + + # Make sure getrawmempool mempool_sequence results aren't "queued" but immediately reflective + # of the time they were gathered. + assert self.nodes[0].getrawmempool(mempool_sequence=True)["mempool_sequence"] > seq_num + + assert_equal((best_hash, "D", None), seq.receive_sequence()) + assert_equal((rbf_txid, "A", seq_num), seq.receive_sequence()) + seq_num += 1 + + # Other things may happen but aren't wallet-deterministic so we don't test for them currently + self.nodes[0].reconsiderblock(best_hash) + self.generatetoaddress(self.nodes[1], 1, ADDRESS_BCRT1_UNSPENDABLE) + + self.log.info("Evict mempool transaction by block conflict") + orig_tx = self.wallet.send_self_transfer(from_node=self.nodes[0]) + orig_txid = orig_tx['txid'] + + # More to be simply mined + more_tx = [] + for _ in range(5): + more_tx.append(self.wallet.send_self_transfer(from_node=self.nodes[0])) + + orig_tx['tx'].vout[0].nValue -= 1000 + bump_txid = self.nodes[0].sendrawtransaction(orig_tx['tx'].serialize().hex()) + # Mine the pre-bump tx + txs_to_add = [orig_tx['hex']] + [tx['hex'] for tx in more_tx] + block = create_block(int(self.nodes[0].getbestblockhash(), 16), create_coinbase(self.nodes[0].getblockcount()+1), txlist=txs_to_add) + add_witness_commitment(block) + block.solve() + assert_equal(self.nodes[0].submitblock(block.serialize().hex()), None) + tip = self.nodes[0].getbestblockhash() + assert_equal(int(tip, 16), block.sha256) + orig_txid_2 = self.wallet.send_self_transfer(from_node=self.nodes[0])['txid'] + + # Flush old notifications until evicted tx original entry + (hash_str, label, mempool_seq) = seq.receive_sequence() + while hash_str != orig_txid: (hash_str, label, mempool_seq) = seq.receive_sequence() - while hash_str != orig_txid: - (hash_str, label, mempool_seq) = seq.receive_sequence() - mempool_seq += 1 + mempool_seq += 1 - # Added original tx - assert_equal(label, "A") - # More transactions to be simply mined - for i in range(len(more_tx)): - assert_equal((more_tx[i], "A", mempool_seq), seq.receive_sequence()) - mempool_seq += 1 - # Bumped by rbf - assert_equal((orig_txid, "R", mempool_seq), seq.receive_sequence()) - mempool_seq += 1 - assert_equal((bump_info["txid"], "A", mempool_seq), seq.receive_sequence()) + # Added original tx + assert_equal(label, "A") + # More transactions to be simply mined + for i in range(len(more_tx)): + assert_equal((more_tx[i]['txid'], "A", mempool_seq), seq.receive_sequence()) mempool_seq += 1 - # Conflict announced first, then block - assert_equal((bump_info["txid"], "R", mempool_seq), seq.receive_sequence()) - mempool_seq += 1 - assert_equal((tip, "C", None), seq.receive_sequence()) - mempool_seq += len(more_tx) - # Last tx - assert_equal((orig_txid_2, "A", mempool_seq), seq.receive_sequence()) - mempool_seq += 1 - self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) - self.sync_all() # want to make sure we didn't break "consensus" for other tests + # Bumped by rbf + assert_equal((orig_txid, "R", mempool_seq), seq.receive_sequence()) + mempool_seq += 1 + assert_equal((bump_txid, "A", mempool_seq), seq.receive_sequence()) + mempool_seq += 1 + # Conflict announced first, then block + assert_equal((bump_txid, "R", mempool_seq), seq.receive_sequence()) + mempool_seq += 1 + assert_equal((tip, "C", None), seq.receive_sequence()) + mempool_seq += len(more_tx) + # Last tx + assert_equal((orig_txid_2, "A", mempool_seq), seq.receive_sequence()) + mempool_seq += 1 + self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) + self.sync_all() # want to make sure we didn't break "consensus" for other tests def test_mempool_sync(self): """ Use sequence notification plus getrawmempool sequence results to "sync mempool" """ - if not self.is_wallet_compiled(): - self.log.info("Skipping mempool sync test") - return self.log.info("Testing 'mempool sync' usage of sequence notifier") [seq] = self.setup_zmq_test([("sequence", "tcp://127.0.0.1:28333")]) @@ -455,10 +452,10 @@ class ZMQTest (BitcoinTestFramework): # Some transactions have been happening but we aren't consuming zmq notifications yet # or we lost a ZMQ message somehow and want to start over - txids = [] + txs = [] num_txs = 5 for _ in range(num_txs): - txids.append(self.nodes[1].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=1.0, replaceable=True)) + txs.append(self.wallet.send_self_transfer(from_node=self.nodes[1])) self.sync_all() # 1) Consume backlog until we get a mempool sequence number @@ -484,11 +481,12 @@ class ZMQTest (BitcoinTestFramework): # Things continue to happen in the "interim" while waiting for snapshot results # We have node 0 do all these to avoid p2p races with RBF announcements for _ in range(num_txs): - txids.append(self.nodes[0].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=0.1, replaceable=True)) - self.nodes[0].bumpfee(txids[-1]) + txs.append(self.wallet.send_self_transfer(from_node=self.nodes[0])) + txs[-1]['tx'].vout[0].nValue -= 1000 + self.nodes[0].sendrawtransaction(txs[-1]['tx'].serialize().hex()) self.sync_all() self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) - final_txid = self.nodes[0].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=0.1, replaceable=True) + final_txid = self.wallet.send_self_transfer(from_node=self.nodes[0])['txid'] # 3) Consume ZMQ backlog until we get to "now" for the mempool snapshot while True: diff --git a/test/functional/mempool_package_onemore.py b/test/functional/mempool_package_onemore.py index a6fb1dcf35..423a5bf2ee 100755 --- a/test/functional/mempool_package_onemore.py +++ b/test/functional/mempool_package_onemore.py @@ -7,74 +7,68 @@ size. """ -from decimal import Decimal - -from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, - chain_transaction, ) +from test_framework.wallet import MiniWallet + MAX_ANCESTORS = 25 MAX_DESCENDANTS = 25 + class MempoolPackagesTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.extra_args = [["-maxorphantx=1000"]] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + def chain_tx(self, utxos_to_spend, *, num_outputs=1): + return self.wallet.send_self_transfer_multi( + from_node=self.nodes[0], + utxos_to_spend=utxos_to_spend, + num_outputs=num_outputs)['new_utxos'] def run_test(self): - # Mine some blocks and have them mature. - self.generate(self.nodes[0], COINBASE_MATURITY + 1) - utxo = self.nodes[0].listunspent(10) - txid = utxo[0]['txid'] - vout = utxo[0]['vout'] - value = utxo[0]['amount'] + self.wallet = MiniWallet(self.nodes[0]) + self.wallet.rescan_utxos() - fee = Decimal("0.0002") # MAX_ANCESTORS transactions off a confirmed tx should be fine chain = [] + utxo = self.wallet.get_utxo() for _ in range(4): - (txid, sent_value) = chain_transaction(self.nodes[0], [txid], [vout], value, fee, 2) - vout = 0 - value = sent_value - chain.append([txid, value]) + utxo, utxo2 = self.chain_tx([utxo], num_outputs=2) + chain.append(utxo2) for _ in range(MAX_ANCESTORS - 4): - (txid, sent_value) = chain_transaction(self.nodes[0], [txid], [0], value, fee, 1) - value = sent_value - chain.append([txid, value]) - (second_chain, second_chain_value) = chain_transaction(self.nodes[0], [utxo[1]['txid']], [utxo[1]['vout']], utxo[1]['amount'], fee, 1) + utxo, = self.chain_tx([utxo]) + chain.append(utxo) + second_chain, = self.chain_tx([self.wallet.get_utxo()]) # Check mempool has MAX_ANCESTORS + 1 transactions in it assert_equal(len(self.nodes[0].getrawmempool()), MAX_ANCESTORS + 1) # Adding one more transaction on to the chain should fail. - assert_raises_rpc_error(-26, "too-long-mempool-chain, too many unconfirmed ancestors [limit: 25]", chain_transaction, self.nodes[0], [txid], [0], value, fee, 1) + assert_raises_rpc_error(-26, "too-long-mempool-chain, too many unconfirmed ancestors [limit: 25]", self.chain_tx, [utxo]) # ...even if it chains on from some point in the middle of the chain. - assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", chain_transaction, self.nodes[0], [chain[2][0]], [1], chain[2][1], fee, 1) - assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", chain_transaction, self.nodes[0], [chain[1][0]], [1], chain[1][1], fee, 1) + assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[2]]) + assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[1]]) # ...even if it chains on to two parent transactions with one in the chain. - assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", chain_transaction, self.nodes[0], [chain[0][0], second_chain], [1, 0], chain[0][1] + second_chain_value, fee, 1) + assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[0], second_chain]) # ...especially if its > 40k weight - assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", chain_transaction, self.nodes[0], [chain[0][0]], [1], chain[0][1], fee, 350) + assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[0]], num_outputs=350) # But not if it chains directly off the first transaction - (replacable_txid, replacable_orig_value) = chain_transaction(self.nodes[0], [chain[0][0]], [1], chain[0][1], fee, 1) + replacable_tx = self.wallet.send_self_transfer_multi(from_node=self.nodes[0], utxos_to_spend=[chain[0]])['tx'] # and the second chain should work just fine - chain_transaction(self.nodes[0], [second_chain], [0], second_chain_value, fee, 1) + self.chain_tx([second_chain]) # Make sure we can RBF the chain which used our carve-out rule - second_tx_outputs = {self.nodes[0].getrawtransaction(replacable_txid, True)["vout"][0]['scriptPubKey']['address']: replacable_orig_value - (Decimal(1) / Decimal(100))} - second_tx = self.nodes[0].createrawtransaction([{'txid': chain[0][0], 'vout': 1}], second_tx_outputs) - signed_second_tx = self.nodes[0].signrawtransactionwithwallet(second_tx) - self.nodes[0].sendrawtransaction(signed_second_tx['hex']) + replacable_tx.vout[0].nValue -= 1000000 + self.nodes[0].sendrawtransaction(replacable_tx.serialize().hex()) # Finally, check that we added two transactions assert_equal(len(self.nodes[0].getrawmempool()), MAX_ANCESTORS + 3) + if __name__ == '__main__': MempoolPackagesTest().main() diff --git a/test/functional/mempool_unbroadcast.py b/test/functional/mempool_unbroadcast.py index adf7326dac..37ef4a9157 100755 --- a/test/functional/mempool_unbroadcast.py +++ b/test/functional/mempool_unbroadcast.py @@ -9,21 +9,20 @@ import time from test_framework.p2p import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_equal, - create_confirmed_utxos, -) +from test_framework.util import assert_equal +from test_framework.wallet import MiniWallet MAX_INITIAL_BROADCAST_DELAY = 15 * 60 # 15 minutes in seconds class MempoolUnbroadcastTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + if self.is_wallet_compiled(): + self.requires_wallet = True def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + self.wallet.rescan_utxos() self.test_broadcast() self.test_txn_removal() @@ -31,30 +30,25 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): self.log.info("Test that mempool reattempts delivery of locally submitted transaction") node = self.nodes[0] - min_relay_fee = node.getnetworkinfo()["relayfee"] - utxos = create_confirmed_utxos(self, min_relay_fee, node, 10) - self.disconnect_nodes(0, 1) self.log.info("Generate transactions that only node 0 knows about") - # generate a wallet txn - addr = node.getnewaddress() - wallet_tx_hsh = node.sendtoaddress(addr, 0.0001) + if self.is_wallet_compiled(): + # generate a wallet txn + addr = node.getnewaddress() + wallet_tx_hsh = node.sendtoaddress(addr, 0.0001) # generate a txn using sendrawtransaction - us0 = utxos.pop() - inputs = [{"txid": us0["txid"], "vout": us0["vout"]}] - outputs = {addr: 0.0001} - tx = node.createrawtransaction(inputs, outputs) - node.settxfee(min_relay_fee) - txF = node.fundrawtransaction(tx) - txFS = node.signrawtransactionwithwallet(txF["hex"]) + txFS = self.wallet.create_self_transfer(from_node=node) rpc_tx_hsh = node.sendrawtransaction(txFS["hex"]) # check transactions are in unbroadcast using rpc mempoolinfo = self.nodes[0].getmempoolinfo() - assert_equal(mempoolinfo['unbroadcastcount'], 2) + unbroadcast_count = 1 + if self.is_wallet_compiled(): + unbroadcast_count += 1 + assert_equal(mempoolinfo['unbroadcastcount'], unbroadcast_count) mempool = self.nodes[0].getrawmempool(True) for tx in mempool: assert_equal(mempool[tx]['unbroadcast'], True) @@ -62,7 +56,8 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): # check that second node doesn't have these two txns mempool = self.nodes[1].getrawmempool() assert rpc_tx_hsh not in mempool - assert wallet_tx_hsh not in mempool + if self.is_wallet_compiled(): + assert wallet_tx_hsh not in mempool # ensure that unbroadcast txs are persisted to mempool.dat self.restart_node(0) @@ -75,7 +70,8 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): self.sync_mempools(timeout=30) mempool = self.nodes[1].getrawmempool() assert rpc_tx_hsh in mempool - assert wallet_tx_hsh in mempool + if self.is_wallet_compiled(): + assert wallet_tx_hsh in mempool # check that transactions are no longer in first node's unbroadcast set mempool = self.nodes[0].getrawmempool(True) @@ -102,8 +98,7 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): # since the node doesn't have any connections, it will not receive # any GETDATAs & thus the transaction will remain in the unbroadcast set. - addr = node.getnewaddress() - txhsh = node.sendtoaddress(addr, 0.0001) + txhsh = self.wallet.send_self_transfer(from_node=node)["txid"] # check transaction was removed from unbroadcast set due to presence in # a block diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index 6f2ac805a0..a15fbe5a24 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -4,11 +4,15 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the prioritisetransaction mining RPC.""" +from decimal import Decimal import time +from test_framework.blocktools import COINBASE_MATURITY from test_framework.messages import COIN, MAX_BLOCK_WEIGHT from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error, create_confirmed_utxos, create_lots_of_big_transactions, gen_return_txouts +from test_framework.wallet import MiniWallet + class PrioritiseTransactionTest(BitcoinTestFramework): def set_test_params(self): @@ -23,7 +27,84 @@ class PrioritiseTransactionTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() + def test_diamond(self): + self.log.info("Test diamond-shape package with priority") + self.generate(self.wallet, COINBASE_MATURITY + 1) + mock_time = int(time.time()) + self.nodes[0].setmocktime(mock_time) + + # tx_a + # / \ + # / \ + # tx_b tx_c + # \ / + # \ / + # tx_d + + tx_o_a = self.wallet.send_self_transfer_multi( + from_node=self.nodes[0], + num_outputs=2, + ) + txid_a = tx_o_a["txid"] + + tx_o_b, tx_o_c = [self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=u, + ) for u in tx_o_a["new_utxos"]] + txid_b = tx_o_b["txid"] + txid_c = tx_o_c["txid"] + + tx_o_d = self.wallet.send_self_transfer_multi( + from_node=self.nodes[0], + utxos_to_spend=[ + self.wallet.get_utxo(txid=txid_b), + self.wallet.get_utxo(txid=txid_c), + ], + ) + txid_d = tx_o_d["txid"] + + self.log.info("Test priority while txs are in mempool") + raw_before = self.nodes[0].getrawmempool(verbose=True) + fee_delta_b = Decimal(9999) / COIN + fee_delta_c_1 = Decimal(-1234) / COIN + fee_delta_c_2 = Decimal(8888) / COIN + self.nodes[0].prioritisetransaction(txid=txid_b, fee_delta=int(fee_delta_b * COIN)) + self.nodes[0].prioritisetransaction(txid=txid_c, fee_delta=int(fee_delta_c_1 * COIN)) + self.nodes[0].prioritisetransaction(txid=txid_c, fee_delta=int(fee_delta_c_2 * COIN)) + raw_before[txid_a]["fees"]["descendant"] += fee_delta_b + fee_delta_c_1 + fee_delta_c_2 + raw_before[txid_b]["fees"]["modified"] += fee_delta_b + raw_before[txid_b]["fees"]["ancestor"] += fee_delta_b + raw_before[txid_b]["fees"]["descendant"] += fee_delta_b + raw_before[txid_c]["fees"]["modified"] += fee_delta_c_1 + fee_delta_c_2 + raw_before[txid_c]["fees"]["ancestor"] += fee_delta_c_1 + fee_delta_c_2 + raw_before[txid_c]["fees"]["descendant"] += fee_delta_c_1 + fee_delta_c_2 + raw_before[txid_d]["fees"]["ancestor"] += fee_delta_b + fee_delta_c_1 + fee_delta_c_2 + raw_after = self.nodes[0].getrawmempool(verbose=True) + assert_equal(raw_before[txid_a], raw_after[txid_a]) + assert_equal(raw_before, raw_after) + + self.log.info("Test priority while txs are not in mempool") + self.restart_node(0, extra_args=["-nopersistmempool"]) + self.nodes[0].setmocktime(mock_time) + assert_equal(self.nodes[0].getmempoolinfo()["size"], 0) + self.nodes[0].prioritisetransaction(txid=txid_b, fee_delta=int(fee_delta_b * COIN)) + self.nodes[0].prioritisetransaction(txid=txid_c, fee_delta=int(fee_delta_c_1 * COIN)) + self.nodes[0].prioritisetransaction(txid=txid_c, fee_delta=int(fee_delta_c_2 * COIN)) + for t in [tx_o_a["hex"], tx_o_b["hex"], tx_o_c["hex"], tx_o_d["hex"]]: + self.nodes[0].sendrawtransaction(t) + raw_after = self.nodes[0].getrawmempool(verbose=True) + assert_equal(raw_before[txid_a], raw_after[txid_a]) + assert_equal(raw_before, raw_after) + + # Clear mempool + self.generate(self.nodes[0], 1) + + # Use default extra_args + self.restart_node(0) + def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + # Test `prioritisetransaction` required parameters assert_raises_rpc_error(-1, "prioritisetransaction", self.nodes[0].prioritisetransaction) assert_raises_rpc_error(-1, "prioritisetransaction", self.nodes[0].prioritisetransaction, '') @@ -44,6 +125,8 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # Test `prioritisetransaction` invalid `fee_delta` assert_raises_rpc_error(-1, "JSON value is not an integer as expected", self.nodes[0].prioritisetransaction, txid=txid, fee_delta='foo') + self.test_diamond() + self.txouts = gen_return_txouts() self.relayfee = self.nodes[0].getnetworkinfo()['relayfee'] diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index 3218a9b14a..e2e9b6dcb2 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -21,8 +21,19 @@ from test_framework.p2p import ( P2P_SERVICES, ) from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_greater_than +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_greater_than_or_equal +) + +ONE_MINUTE = 60 +TEN_MINUTES = 10 * ONE_MINUTE +ONE_HOUR = 60 * ONE_MINUTE +TWO_HOURS = 2 * ONE_HOUR +ONE_DAY = 24 * ONE_HOUR +ADDR_DESTINATIONS_THRESHOLD = 4 class AddrReceiver(P2PInterface): num_ipv4_received = 0 @@ -85,6 +96,9 @@ class AddrTest(BitcoinTestFramework): self.relay_tests() self.inbound_blackhole_tests() + self.destination_rotates_once_in_24_hours_test() + self.destination_rotates_more_than_once_over_several_days_test() + # This test populates the addrman, which can impact the node's behavior # in subsequent tests self.getaddr_tests() @@ -362,6 +376,56 @@ class AddrTest(BitcoinTestFramework): self.nodes[0].disconnect_p2ps() + def get_nodes_that_received_addr(self, peer, receiver_peer, addr_receivers, + time_interval_1, time_interval_2): + + # Clean addr response related to the initial getaddr. There is no way to avoid initial + # getaddr because the peer won't self-announce then. + for addr_receiver in addr_receivers: + addr_receiver.num_ipv4_received = 0 + + for _ in range(10): + self.mocktime += time_interval_1 + self.msg.addrs[0].time = self.mocktime + TEN_MINUTES + self.nodes[0].setmocktime(self.mocktime) + with self.nodes[0].assert_debug_log(['received: addr (31 bytes) peer=0']): + peer.send_and_ping(self.msg) + self.mocktime += time_interval_2 + self.nodes[0].setmocktime(self.mocktime) + receiver_peer.sync_with_ping() + return [node for node in addr_receivers if node.addr_received()] + + def destination_rotates_once_in_24_hours_test(self): + self.restart_node(0, []) + + self.log.info('Test within 24 hours an addr relay destination is rotated at most once') + self.mocktime = int(time.time()) + self.msg = self.setup_addr_msg(1) + self.addr_receivers = [] + peer = self.nodes[0].add_p2p_connection(P2PInterface()) + receiver_peer = self.nodes[0].add_p2p_connection(AddrReceiver()) + addr_receivers = [self.nodes[0].add_p2p_connection(AddrReceiver()) for _ in range(20)] + nodes_received_addr = self.get_nodes_that_received_addr(peer, receiver_peer, addr_receivers, 0, TWO_HOURS) # 10 intervals of 2 hours + # Per RelayAddress, we would announce these addrs to 2 destinations per day. + # Since it's at most one rotation, at most 4 nodes can receive ADDR. + assert_greater_than_or_equal(ADDR_DESTINATIONS_THRESHOLD, len(nodes_received_addr)) + self.nodes[0].disconnect_p2ps() + + def destination_rotates_more_than_once_over_several_days_test(self): + self.restart_node(0, []) + + self.log.info('Test after several days an addr relay destination is rotated more than once') + self.msg = self.setup_addr_msg(1) + peer = self.nodes[0].add_p2p_connection(P2PInterface()) + receiver_peer = self.nodes[0].add_p2p_connection(AddrReceiver()) + addr_receivers = [self.nodes[0].add_p2p_connection(AddrReceiver()) for _ in range(20)] + # 10 intervals of 1 day (+ 1 hour, which should be enough to cover 30-min Poisson in most cases) + nodes_received_addr = self.get_nodes_that_received_addr(peer, receiver_peer, addr_receivers, ONE_DAY, ONE_HOUR) + # Now that there should have been more than one rotation, more than + # ADDR_DESTINATIONS_THRESHOLD nodes should have received ADDR. + assert_greater_than(len(nodes_received_addr), ADDR_DESTINATIONS_THRESHOLD) + self.nodes[0].disconnect_p2ps() + if __name__ == '__main__': AddrTest().main() diff --git a/test/functional/p2p_blockfilters.py b/test/functional/p2p_blockfilters.py index e73fad439f..2ffc825acd 100755 --- a/test/functional/p2p_blockfilters.py +++ b/test/functional/p2p_blockfilters.py @@ -244,6 +244,18 @@ class CompactFiltersTest(BitcoinTestFramework): peer_0.send_message(request) peer_0.wait_for_disconnect() + self.log.info("Test -peerblockfilters without -blockfilterindex raises an error") + self.stop_node(0) + self.nodes[0].extra_args = ["-peerblockfilters"] + msg = "Error: Cannot set -peerblockfilters without -blockfilterindex." + self.nodes[0].assert_start_raises_init_error(expected_msg=msg) + + self.log.info("Test -blockfilterindex with -reindex-chainstate raises an error") + self.nodes[0].assert_start_raises_init_error( + expected_msg='Error: -reindex-chainstate option is not compatible with -blockfilterindex. ' + 'Please temporarily disable blockfilterindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.', + extra_args=['-blockfilterindex', '-reindex-chainstate'], + ) def compute_last_header(prev_header, hashes): """Compute the last filter header from a starting header and a sequence of filter hashes.""" diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index 6f142f23f2..12ee4b3c27 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -94,7 +94,7 @@ class P2PBlocksOnly(BitcoinTestFramework): self.nodes[0].sendrawtransaction(tx_hex) - # Bump time forward to ensure nNextInvSend timer pops + # Bump time forward to ensure m_next_inv_send_time timer pops self.nodes[0].setmocktime(int(time.time()) + 60) conn.sync_send_with_ping() diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index f377fbaaa6..89ddfd3bcf 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -43,7 +43,6 @@ from test_framework.messages import ( ser_uint256, ser_vector, sha256, - tx_from_hex, ) from test_framework.p2p import ( P2PInterface, @@ -89,6 +88,8 @@ from test_framework.util import ( softfork_active, assert_raises_rpc_error, ) +from test_framework.wallet import MiniWallet + MAX_SIGOP_COST = 80000 @@ -221,9 +222,6 @@ class SegWitTest(BitcoinTestFramework): ] self.supports_cli = False - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - # Helper functions def build_next_block(self): @@ -259,6 +257,7 @@ class SegWitTest(BitcoinTestFramework): self.log.info("Starting tests before segwit activation") self.segwit_active = False + self.wallet = MiniWallet(self.nodes[0]) self.test_non_witness_transaction() self.test_v0_outputs_arent_spendable() @@ -307,7 +306,7 @@ class SegWitTest(BitcoinTestFramework): self.test_node.send_and_ping(msg_no_witness_block(block)) # make sure the block was processed txid = block.vtx[0].sha256 - self.generate(self.nodes[0], 99) # let the block mature + self.generate(self.wallet, 99) # let the block mature # Create a transaction that spends the coinbase tx = CTransaction() @@ -1999,21 +1998,13 @@ class SegWitTest(BitcoinTestFramework): def serialize(self): return serialize_with_bogus_witness(self.tx) - self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(address_type='bech32'), 5) - self.generate(self.nodes[0], 1) - unspent = next(u for u in self.nodes[0].listunspent() if u['spendable'] and u['address'].startswith('bcrt')) - - raw = self.nodes[0].createrawtransaction([{"txid": unspent['txid'], "vout": unspent['vout']}], {self.nodes[0].getnewaddress(): 1}) - tx = tx_from_hex(raw) + tx = self.wallet.create_self_transfer(from_node=self.nodes[0])['tx'] assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].decoderawtransaction, hexstring=serialize_with_bogus_witness(tx).hex(), iswitness=True) - with self.nodes[0].assert_debug_log(['Superfluous witness record']): + with self.nodes[0].assert_debug_log(['Unknown transaction optional data']): self.test_node.send_and_ping(msg_bogus_tx(tx)) - raw = self.nodes[0].signrawtransactionwithwallet(raw) - assert raw['complete'] - raw = raw['hex'] - tx = tx_from_hex(raw) + tx.wit.vtxinwit = [] # drop witness assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].decoderawtransaction, hexstring=serialize_with_bogus_witness(tx).hex(), iswitness=True) - with self.nodes[0].assert_debug_log(['Unknown transaction optional data']): + with self.nodes[0].assert_debug_log(['Superfluous witness record']): self.test_node.send_and_ping(msg_bogus_tx(tx)) @subtest diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index b264f23fb5..193bd3f1cd 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -200,7 +200,7 @@ class BlockchainTest(BitcoinTestFramework): 'timeout': 0x7fffffffffffffff, # testdummy does not have a timeout so is set to the max int64 value 'min_activation_height': 0, 'status': 'started', - 'status-next': status_next, + 'status_next': status_next, 'since': 144, 'statistics': { 'period': 144, @@ -220,7 +220,7 @@ class BlockchainTest(BitcoinTestFramework): 'timeout': 9223372036854775807, 'min_activation_height': 0, 'status': 'active', - 'status-next': 'active', + 'status_next': 'active', 'since': 0, }, 'height': 0, diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index 1a3d14100f..1695acaaa8 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -18,15 +18,18 @@ from test_framework.util import ( assert_equal, ) from test_framework.wallet_util import bytes_to_wif +from test_framework.wallet import ( + MiniWallet, + getnewdestination, +) class RpcCreateMultiSigTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 self.supports_cli = False - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + if self.is_bdb_compiled(): + self.requires_wallet = True def get_keys(self): self.pub = [] @@ -37,15 +40,20 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): k.generate() self.pub.append(k.get_pubkey().get_bytes().hex()) self.priv.append(bytes_to_wif(k.get_bytes(), k.is_compressed)) - self.final = node2.getnewaddress() + if self.is_bdb_compiled(): + self.final = node2.getnewaddress() + else: + self.final = getnewdestination()[2] def run_test(self): node0, node1, node2 = self.nodes + self.wallet = MiniWallet(test_node=node0) - self.check_addmultisigaddress_errors() + if self.is_bdb_compiled(): + self.check_addmultisigaddress_errors() self.log.info('Generating blocks ...') - self.generate(node0, 149) + self.generate(self.wallet, 149) self.moved = 0 for self.nkeys in [3, 5]: @@ -53,14 +61,14 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): for self.output_type in ["bech32", "p2sh-segwit", "legacy"]: self.get_keys() self.do_multisig() - - self.checkbalances() + if self.is_bdb_compiled(): + self.checkbalances() # Test mixed compressed and uncompressed pubkeys self.log.info('Mixed compressed and uncompressed multisigs are not allowed') - pk0 = node0.getaddressinfo(node0.getnewaddress())['pubkey'] - pk1 = node1.getaddressinfo(node1.getnewaddress())['pubkey'] - pk2 = node2.getaddressinfo(node2.getnewaddress())['pubkey'] + pk0 = getnewdestination()[0].hex() + pk1 = getnewdestination()[0].hex() + pk2 = getnewdestination()[0].hex() # decompress pk2 pk_obj = ECPubKey() @@ -68,26 +76,30 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): pk_obj.compressed = False pk2 = pk_obj.get_bytes().hex() - node0.createwallet(wallet_name='wmulti0', disable_private_keys=True) - wmulti0 = node0.get_wallet_rpc('wmulti0') + if self.is_bdb_compiled(): + node0.createwallet(wallet_name='wmulti0', disable_private_keys=True) + wmulti0 = node0.get_wallet_rpc('wmulti0') # Check all permutations of keys because order matters apparently for keys in itertools.permutations([pk0, pk1, pk2]): # Results should be the same as this legacy one legacy_addr = node0.createmultisig(2, keys, 'legacy')['address'] - result = wmulti0.addmultisigaddress(2, keys, '', 'legacy') - assert_equal(legacy_addr, result['address']) - assert 'warnings' not in result + + if self.is_bdb_compiled(): + result = wmulti0.addmultisigaddress(2, keys, '', 'legacy') + assert_equal(legacy_addr, result['address']) + assert 'warnings' not in result # Generate addresses with the segwit types. These should all make legacy addresses for addr_type in ['bech32', 'p2sh-segwit']: - result = wmulti0.createmultisig(2, keys, addr_type) + result = self.nodes[0].createmultisig(2, keys, addr_type) assert_equal(legacy_addr, result['address']) assert_equal(result['warnings'], ["Unable to make chosen address type, please ensure no uncompressed public keys are present."]) - result = wmulti0.addmultisigaddress(2, keys, '', addr_type) - assert_equal(legacy_addr, result['address']) - assert_equal(result['warnings'], ["Unable to make chosen address type, please ensure no uncompressed public keys are present."]) + if self.is_bdb_compiled(): + result = wmulti0.addmultisigaddress(2, keys, '', addr_type) + assert_equal(legacy_addr, result['address']) + assert_equal(result['warnings'], ["Unable to make chosen address type, please ensure no uncompressed public keys are present."]) self.log.info('Testing sortedmulti descriptors with BIP 67 test vectors') with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/rpc_bip67.json'), encoding='utf-8') as f: @@ -126,26 +138,29 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): bal0 = node0.getbalance() bal1 = node1.getbalance() bal2 = node2.getbalance() + balw = self.wallet.get_balance() height = node0.getblockchaininfo()["blocks"] assert 150 < height < 350 total = 149 * 50 + (height - 149 - 100) * 25 assert bal1 == 0 assert bal2 == self.moved - assert bal0 + bal1 + bal2 == total + assert_equal(bal0 + bal1 + bal2 + balw, total) def do_multisig(self): node0, node1, node2 = self.nodes - if 'wmulti' not in node1.listwallets(): - try: - node1.loadwallet('wmulti') - except JSONRPCException as e: - path = os.path.join(self.options.tmpdir, "node1", "regtest", "wallets", "wmulti") - if e.error['code'] == -18 and "Wallet file verification failed. Failed to load database path '{}'. Path does not exist.".format(path) in e.error['message']: - node1.createwallet(wallet_name='wmulti', disable_private_keys=True) - else: - raise - wmulti = node1.get_wallet_rpc('wmulti') + + if self.is_bdb_compiled(): + if 'wmulti' not in node1.listwallets(): + try: + node1.loadwallet('wmulti') + except JSONRPCException as e: + path = os.path.join(self.options.tmpdir, "node1", "regtest", "wallets", "wmulti") + if e.error['code'] == -18 and "Wallet file verification failed. Failed to load database path '{}'. Path does not exist.".format(path) in e.error['message']: + node1.createwallet(wallet_name='wmulti', disable_private_keys=True) + else: + raise + wmulti = node1.get_wallet_rpc('wmulti') # Construct the expected descriptor desc = 'multi({},{})'.format(self.nsigs, ','.join(self.pub)) @@ -164,17 +179,19 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): if self.output_type == 'bech32': assert madd[0:4] == "bcrt" # actually a bech32 address - # compare against addmultisigaddress - msigw = wmulti.addmultisigaddress(self.nsigs, self.pub, None, self.output_type) - maddw = msigw["address"] - mredeemw = msigw["redeemScript"] - assert_equal(desc, drop_origins(msigw['descriptor'])) - # addmultisigiaddress and createmultisig work the same - assert maddw == madd - assert mredeemw == mredeem - - txid = node0.sendtoaddress(madd, 40) - + if self.is_bdb_compiled(): + # compare against addmultisigaddress + msigw = wmulti.addmultisigaddress(self.nsigs, self.pub, None, self.output_type) + maddw = msigw["address"] + mredeemw = msigw["redeemScript"] + assert_equal(desc, drop_origins(msigw['descriptor'])) + # addmultisigiaddress and createmultisig work the same + assert maddw == madd + assert mredeemw == mredeem + wmulti.unloadwallet() + + spk = bytes.fromhex(node0.validateaddress(madd)["scriptPubKey"]) + txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=spk, amount=1300) tx = node0.getrawtransaction(txid, True) vout = [v["n"] for v in tx["vout"] if madd == v["scriptPubKey"]["address"]] assert len(vout) == 1 @@ -225,8 +242,6 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): txinfo = node0.getrawtransaction(tx, True, blk) self.log.info("n/m=%d/%d %s size=%d vsize=%d weight=%d" % (self.nsigs, self.nkeys, self.output_type, txinfo["size"], txinfo["vsize"], txinfo["weight"])) - wmulti.unloadwallet() - if __name__ == '__main__': RpcCreateMultiSigTest().main() diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py index 1721b6ffe8..672c9a53dc 100755 --- a/test/functional/rpc_dumptxoutset.py +++ b/test/functional/rpc_dumptxoutset.py @@ -37,21 +37,25 @@ class DumptxoutsetTest(BitcoinTestFramework): # Blockhash should be deterministic based on mocked time. assert_equal( out['base_hash'], - '6fd417acba2a8738b06fee43330c50d58e6a725046c3d843c8dd7e51d46d1ed6') + '09abf0e7b510f61ca6cf33bab104e9ee99b3528b371d27a2d4b39abb800fba7e') with open(str(expected_path), 'rb') as f: digest = hashlib.sha256(f.read()).hexdigest() # UTXO snapshot hash should be deterministic based on mocked time. assert_equal( - digest, '7ae82c986fa5445678d2a21453bb1c86d39e47af13da137640c2b1cf8093691c') + digest, 'b1bacb602eacf5fbc9a7c2ef6eeb0d229c04e98bdf0c2ea5929012cd0eae3830') assert_equal( - out['txoutset_hash'], 'd4b614f476b99a6e569973bf1c0120d88b1a168076f8ce25691fb41dd1cef149') + out['txoutset_hash'], '1f7e3befd45dc13ae198dfbb22869a9c5c4196f8e9ef9735831af1288033f890') assert_equal(out['nchaintx'], 101) - # Specifying a path to an existing file will fail. + # Specifying a path to an existing or invalid file will fail. assert_raises_rpc_error( -8, '{} already exists'.format(FILENAME), node.dumptxoutset, FILENAME) + invalid_path = str(Path(node.datadir) / "invalid" / "path") + assert_raises_rpc_error( + -8, "Couldn't open file {}.incomplete for writing".format(invalid_path), node.dumptxoutset, invalid_path) + if __name__ == '__main__': DumptxoutsetTest().main() diff --git a/test/functional/rpc_generate.py b/test/functional/rpc_generate.py index 47d7814da3..2b1dd20ea1 100755 --- a/test/functional/rpc_generate.py +++ b/test/functional/rpc_generate.py @@ -2,9 +2,10 @@ # Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test generate RPC.""" +"""Test generate* RPCs.""" from test_framework.test_framework import BitcoinTestFramework +from test_framework.wallet import MiniWallet from test_framework.util import ( assert_equal, assert_raises_rpc_error, @@ -16,6 +17,94 @@ class RPCGenerateTest(BitcoinTestFramework): self.num_nodes = 1 def run_test(self): + self.test_generatetoaddress() + self.test_generate() + self.test_generateblock() + + def test_generatetoaddress(self): + self.generatetoaddress(self.nodes[0], 1, 'mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ') + assert_raises_rpc_error(-5, "Invalid address", self.generatetoaddress, self.nodes[0], 1, '3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy') + + def test_generateblock(self): + node = self.nodes[0] + miniwallet = MiniWallet(node) + miniwallet.rescan_utxos() + + self.log.info('Generate an empty block to address') + address = miniwallet.get_address() + hash = self.generateblock(node, output=address, transactions=[])['hash'] + block = node.getblock(blockhash=hash, verbose=2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], address) + + self.log.info('Generate an empty block to a descriptor') + hash = self.generateblock(node, 'addr(' + address + ')', [])['hash'] + block = node.getblock(blockhash=hash, verbosity=2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], address) + + self.log.info('Generate an empty block to a combo descriptor with compressed pubkey') + combo_key = '0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798' + combo_address = 'bcrt1qw508d6qejxtdg4y5r3zarvary0c5xw7kygt080' + hash = self.generateblock(node, 'combo(' + combo_key + ')', [])['hash'] + block = node.getblock(hash, 2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], combo_address) + + self.log.info('Generate an empty block to a combo descriptor with uncompressed pubkey') + combo_key = '0408ef68c46d20596cc3f6ddf7c8794f71913add807f1dc55949fa805d764d191c0b7ce6894c126fce0babc6663042f3dde9b0cf76467ea315514e5a6731149c67' + combo_address = 'mkc9STceoCcjoXEXe6cm66iJbmjM6zR9B2' + hash = self.generateblock(node, 'combo(' + combo_key + ')', [])['hash'] + block = node.getblock(hash, 2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], combo_address) + + # Generate some extra mempool transactions to verify they don't get mined + for _ in range(10): + miniwallet.send_self_transfer(from_node=node) + + self.log.info('Generate block with txid') + txid = miniwallet.send_self_transfer(from_node=node)['txid'] + hash = self.generateblock(node, address, [txid])['hash'] + block = node.getblock(hash, 1) + assert_equal(len(block['tx']), 2) + assert_equal(block['tx'][1], txid) + + self.log.info('Generate block with raw tx') + rawtx = miniwallet.create_self_transfer()['hex'] + hash = self.generateblock(node, address, [rawtx])['hash'] + + block = node.getblock(hash, 1) + assert_equal(len(block['tx']), 2) + txid = block['tx'][1] + assert_equal(node.getrawtransaction(txid=txid, verbose=False, blockhash=hash), rawtx) + + self.log.info('Fail to generate block with out of order txs') + txid1 = miniwallet.send_self_transfer(from_node=node)['txid'] + utxo1 = miniwallet.get_utxo(txid=txid1) + rawtx2 = miniwallet.create_self_transfer(utxo_to_spend=utxo1)['hex'] + assert_raises_rpc_error(-25, 'TestBlockValidity failed: bad-txns-inputs-missingorspent', self.generateblock, node, address, [rawtx2, txid1]) + + self.log.info('Fail to generate block with txid not in mempool') + missing_txid = '0000000000000000000000000000000000000000000000000000000000000000' + assert_raises_rpc_error(-5, 'Transaction ' + missing_txid + ' not in mempool.', self.generateblock, node, address, [missing_txid]) + + self.log.info('Fail to generate block with invalid raw tx') + invalid_raw_tx = '0000' + assert_raises_rpc_error(-22, 'Transaction decode failed for ' + invalid_raw_tx, self.generateblock, node, address, [invalid_raw_tx]) + + self.log.info('Fail to generate block with invalid address/descriptor') + assert_raises_rpc_error(-5, 'Invalid address or descriptor', self.generateblock, node, '1234', []) + + self.log.info('Fail to generate block with a ranged descriptor') + ranged_descriptor = 'pkh(tpubD6NzVbkrYhZ4XgiXtGrdW5XDAPFCL9h7we1vwNCpn8tGbBcgfVYjXyhWo4E1xkh56hjod1RhGjxbaTLV3X4FyWuejifB9jusQ46QzG87VKp/0/*)' + assert_raises_rpc_error(-8, 'Ranged descriptor not accepted. Maybe pass through deriveaddresses first?', self.generateblock, node, ranged_descriptor, []) + + self.log.info('Fail to generate block with a descriptor missing a private key') + child_descriptor = 'pkh(tpubD6NzVbkrYhZ4XgiXtGrdW5XDAPFCL9h7we1vwNCpn8tGbBcgfVYjXyhWo4E1xkh56hjod1RhGjxbaTLV3X4FyWuejifB9jusQ46QzG87VKp/0\'/0)' + assert_raises_rpc_error(-5, 'Cannot derive script without private keys', self.generateblock, node, child_descriptor, []) + + def test_generate(self): message = ( "generate\n\n" "has been replaced by the -generate " diff --git a/test/functional/rpc_generateblock.py b/test/functional/rpc_generateblock.py deleted file mode 100755 index 7eeb745817..0000000000 --- a/test/functional/rpc_generateblock.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2020-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -'''Test generateblock rpc. -''' - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.wallet import MiniWallet -from test_framework.util import ( - assert_equal, - assert_raises_rpc_error, -) - - -class GenerateBlockTest(BitcoinTestFramework): - def set_test_params(self): - self.num_nodes = 1 - - def run_test(self): - node = self.nodes[0] - miniwallet = MiniWallet(node) - miniwallet.rescan_utxos() - - self.log.info('Generate an empty block to address') - address = miniwallet.get_address() - hash = self.generateblock(node, output=address, transactions=[])['hash'] - block = node.getblock(blockhash=hash, verbose=2) - assert_equal(len(block['tx']), 1) - assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], address) - - self.log.info('Generate an empty block to a descriptor') - hash = self.generateblock(node, 'addr(' + address + ')', [])['hash'] - block = node.getblock(blockhash=hash, verbosity=2) - assert_equal(len(block['tx']), 1) - assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], address) - - self.log.info('Generate an empty block to a combo descriptor with compressed pubkey') - combo_key = '0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798' - combo_address = 'bcrt1qw508d6qejxtdg4y5r3zarvary0c5xw7kygt080' - hash = self.generateblock(node, 'combo(' + combo_key + ')', [])['hash'] - block = node.getblock(hash, 2) - assert_equal(len(block['tx']), 1) - assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], combo_address) - - self.log.info('Generate an empty block to a combo descriptor with uncompressed pubkey') - combo_key = '0408ef68c46d20596cc3f6ddf7c8794f71913add807f1dc55949fa805d764d191c0b7ce6894c126fce0babc6663042f3dde9b0cf76467ea315514e5a6731149c67' - combo_address = 'mkc9STceoCcjoXEXe6cm66iJbmjM6zR9B2' - hash = self.generateblock(node, 'combo(' + combo_key + ')', [])['hash'] - block = node.getblock(hash, 2) - assert_equal(len(block['tx']), 1) - assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], combo_address) - - # Generate some extra mempool transactions to verify they don't get mined - for _ in range(10): - miniwallet.send_self_transfer(from_node=node) - - self.log.info('Generate block with txid') - txid = miniwallet.send_self_transfer(from_node=node)['txid'] - hash = self.generateblock(node, address, [txid])['hash'] - block = node.getblock(hash, 1) - assert_equal(len(block['tx']), 2) - assert_equal(block['tx'][1], txid) - - self.log.info('Generate block with raw tx') - rawtx = miniwallet.create_self_transfer()['hex'] - hash = self.generateblock(node, address, [rawtx])['hash'] - - block = node.getblock(hash, 1) - assert_equal(len(block['tx']), 2) - txid = block['tx'][1] - assert_equal(node.getrawtransaction(txid=txid, verbose=False, blockhash=hash), rawtx) - - self.log.info('Fail to generate block with out of order txs') - txid1 = miniwallet.send_self_transfer(from_node=node)['txid'] - utxo1 = miniwallet.get_utxo(txid=txid1) - rawtx2 = miniwallet.create_self_transfer(utxo_to_spend=utxo1)['hex'] - assert_raises_rpc_error(-25, 'TestBlockValidity failed: bad-txns-inputs-missingorspent', self.generateblock, node, address, [rawtx2, txid1]) - - self.log.info('Fail to generate block with txid not in mempool') - missing_txid = '0000000000000000000000000000000000000000000000000000000000000000' - assert_raises_rpc_error(-5, 'Transaction ' + missing_txid + ' not in mempool.', self.generateblock, node, address, [missing_txid]) - - self.log.info('Fail to generate block with invalid raw tx') - invalid_raw_tx = '0000' - assert_raises_rpc_error(-22, 'Transaction decode failed for ' + invalid_raw_tx, self.generateblock, node, address, [invalid_raw_tx]) - - self.log.info('Fail to generate block with invalid address/descriptor') - assert_raises_rpc_error(-5, 'Invalid address or descriptor', self.generateblock, node, '1234', []) - - self.log.info('Fail to generate block with a ranged descriptor') - ranged_descriptor = 'pkh(tpubD6NzVbkrYhZ4XgiXtGrdW5XDAPFCL9h7we1vwNCpn8tGbBcgfVYjXyhWo4E1xkh56hjod1RhGjxbaTLV3X4FyWuejifB9jusQ46QzG87VKp/0/*)' - assert_raises_rpc_error(-8, 'Ranged descriptor not accepted. Maybe pass through deriveaddresses first?', self.generateblock, node, ranged_descriptor, []) - - self.log.info('Fail to generate block with a descriptor missing a private key') - child_descriptor = 'pkh(tpubD6NzVbkrYhZ4XgiXtGrdW5XDAPFCL9h7we1vwNCpn8tGbBcgfVYjXyhWo4E1xkh56hjod1RhGjxbaTLV3X4FyWuejifB9jusQ46QzG87VKp/0\'/0)' - assert_raises_rpc_error(-5, 'Cannot derive script without private keys', self.generateblock, node, child_descriptor, []) - -if __name__ == '__main__': - GenerateBlockTest().main() diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py index 2f1796d7cc..f6ee6a5215 100755 --- a/test/functional/rpc_misc.py +++ b/test/functional/rpc_misc.py @@ -27,7 +27,7 @@ class RpcMiscTest(BitcoinTestFramework): self.log.info("test CHECK_NONFATAL") assert_raises_rpc_error( -1, - 'Internal bug detected: \'request.params[9].get_str() != "trigger_internal_bug"\'', + 'Internal bug detected: "request.params[9].get_str() != "trigger_internal_bug""', lambda: node.echo(arg9='trigger_internal_bug'), ) @@ -56,9 +56,6 @@ class RpcMiscTest(BitcoinTestFramework): self.log.info("test logging rpc and help") - # Test logging RPC returns the expected number of logging categories. - assert_equal(len(node.logging()), 27) - # Test toggling a logging category on/off/on with the logging RPC. assert_equal(node.logging()['qt'], True) node.logging(exclude=['qt']) diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index b037807b53..444e56610e 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -10,6 +10,10 @@ from itertools import product from test_framework.descriptors import descsum_create from test_framework.key import ECKey +from test_framework.messages import ( + ser_compact_size, + WITNESS_SCALE_FACTOR, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_approx, @@ -615,8 +619,8 @@ class PSBTTest(BitcoinTestFramework): self.nodes[1].createwallet("extfund") wallet = self.nodes[1].get_wallet_rpc("extfund") - # Make a weird but signable script. sh(pkh()) descriptor accomplishes this - desc = descsum_create("sh(pkh({}))".format(privkey)) + # Make a weird but signable script. sh(wsh(pkh())) descriptor accomplishes this + desc = descsum_create("sh(wsh(pkh({})))".format(privkey)) if self.options.descriptors: res = self.nodes[0].importdescriptors([{"desc": desc, "timestamp": "now"}]) else: @@ -634,7 +638,7 @@ class PSBTTest(BitcoinTestFramework): assert_raises_rpc_error(-4, "Insufficient funds", wallet.walletcreatefundedpsbt, [ext_utxo], {self.nodes[0].getnewaddress(): 15}) # But funding should work when the solving data is provided - psbt = wallet.walletcreatefundedpsbt([ext_utxo], {self.nodes[0].getnewaddress(): 15}, 0, {"add_inputs": True, "solving_data": {"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}}) + psbt = wallet.walletcreatefundedpsbt([ext_utxo], {self.nodes[0].getnewaddress(): 15}, 0, {"add_inputs": True, "solving_data": {"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"], addr_info["embedded"]["embedded"]["scriptPubKey"]]}}) signed = wallet.walletprocesspsbt(psbt['psbt']) assert not signed['complete'] signed = self.nodes[0].walletprocesspsbt(signed['psbt']) @@ -655,10 +659,11 @@ class PSBTTest(BitcoinTestFramework): break psbt_in = dec["inputs"][input_idx] # Calculate the input weight - # (prevout + sequence + length of scriptSig + 2 bytes buffer) * 4 + len of scriptwitness + # (prevout + sequence + length of scriptSig + scriptsig + 1 byte buffer) * WITNESS_SCALE_FACTOR + num scriptWitness stack items + (length of stack item + stack item) * N stack items + 1 byte buffer len_scriptsig = len(psbt_in["final_scriptSig"]["hex"]) // 2 if "final_scriptSig" in psbt_in else 0 - len_scriptwitness = len(psbt_in["final_scriptwitness"]["hex"]) // 2 if "final_scriptwitness" in psbt_in else 0 - input_weight = ((41 + len_scriptsig + 2) * 4) + len_scriptwitness + len_scriptsig += len(ser_compact_size(len_scriptsig)) + 1 + len_scriptwitness = (sum([(len(x) // 2) + len(ser_compact_size(len(x) // 2)) for x in psbt_in["final_scriptwitness"]]) + len(psbt_in["final_scriptwitness"]) + 1) if "final_scriptwitness" in psbt_in else 0 + input_weight = ((40 + len_scriptsig) * WITNESS_SCALE_FACTOR) + len_scriptwitness low_input_weight = input_weight // 2 high_input_weight = input_weight * 2 diff --git a/test/functional/rpc_users.py b/test/functional/rpc_users.py index 7cedb4336b..1a35a57802 100755 --- a/test/functional/rpc_users.py +++ b/test/functional/rpc_users.py @@ -107,6 +107,9 @@ class HTTPBasicsTest(BitcoinTestFramework): self.stop_node(0) self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo']) self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo:bar']) + self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo:bar:baz']) + self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo$bar:baz']) + self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo$bar$baz']) self.log.info('Check that failure to write cookie file will abort the node gracefully') cookie_file = os.path.join(get_datadir_path(self.options.tmpdir, 0), self.chain, '.cookie.tmp') diff --git a/test/functional/test_framework/address.py b/test/functional/test_framework/address.py index c7fbf679b6..fcea24655b 100644 --- a/test/functional/test_framework/address.py +++ b/test/functional/test_framework/address.py @@ -35,7 +35,7 @@ class AddressType(enum.Enum): legacy = 'legacy' # P2PKH -chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' +b58chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' def create_deterministic_address_bcrt1_p2tr_op_true(): @@ -59,10 +59,10 @@ def byte_to_base58(b, version): b += hash256(b)[:4] # append checksum value = int.from_bytes(b, 'big') while value > 0: - result = chars[value % 58] + result + result = b58chars[value % 58] + result value //= 58 while b[0] == 0: - result = chars[0] + result + result = b58chars[0] + result b = b[1:] return result @@ -76,8 +76,8 @@ def base58_to_byte(s): n = 0 for c in s: n *= 58 - assert c in chars - digit = chars.index(c) + assert c in b58chars + digit = b58chars.index(c) n += digit h = '%x' % n if len(h) % 2: @@ -85,14 +85,14 @@ def base58_to_byte(s): res = n.to_bytes((n.bit_length() + 7) // 8, 'big') pad = 0 for c in s: - if c == chars[0]: + if c == b58chars[0]: pad += 1 else: break res = b'\x00' * pad + res - # Assert if the checksum is invalid - assert_equal(hash256(res[:-4])[:4], res[-4:]) + if hash256(res[:-4])[:4] != res[-4:]: + raise ValueError('Invalid Base58Check checksum') return res[1:-4], int(res[0]) diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 7791ae5392..2b70eab4e4 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -27,6 +27,7 @@ from .messages import ( from .ripemd160 import ripemd160 MAX_SCRIPT_ELEMENT_SIZE = 520 +MAX_PUBKEYS_PER_MULTI_A = 999 LOCKTIME_THRESHOLD = 500000000 ANNEX_TAG = 0x50 diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index ecdc3bf424..a39ee003ef 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -9,6 +9,7 @@ from enum import Enum import argparse import logging import os +import platform import pdb import random import re @@ -243,8 +244,14 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): "src", "bitcoin-cli" + config["environment"]["EXEEXT"], ) + fname_bitcoinutil = os.path.join( + config["environment"]["BUILDDIR"], + "src", + "bitcoin-util" + config["environment"]["EXEEXT"], + ) self.options.bitcoind = os.getenv("BITCOIND", default=fname_bitcoind) self.options.bitcoincli = os.getenv("BITCOINCLI", default=fname_bitcoincli) + self.options.bitcoinutil = os.getenv("BITCOINUTIL", default=fname_bitcoinutil) os.environ['PATH'] = os.pathsep.join([ os.path.join(config['environment']['BUILDDIR'], 'src'), @@ -826,6 +833,29 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): except ImportError: raise SkipTest("python3-zmq module not available.") + def skip_if_no_python_bcc(self): + """Attempt to import the bcc package and skip the tests if the import fails.""" + try: + import bcc # type: ignore[import] # noqa: F401 + except ImportError: + raise SkipTest("bcc python module not available") + + def skip_if_no_bitcoind_tracepoints(self): + """Skip the running test if bitcoind has not been compiled with USDT tracepoint support.""" + if not self.is_usdt_compiled(): + raise SkipTest("bitcoind has not been built with USDT tracepoints enabled.") + + def skip_if_no_bpf_permissions(self): + """Skip the running test if we don't have permissions to do BPF syscalls and load BPF maps.""" + # check for 'root' permissions + if os.geteuid() != 0: + raise SkipTest("no permissions to use BPF (please review the tests carefully before running them with higher privileges)") + + def skip_if_platform_not_linux(self): + """Skip the running test if we are not on a Linux platform""" + if platform.system() != "Linux": + raise SkipTest("not on a Linux system") + def skip_if_no_bitcoind_zmq(self): """Skip the running test if bitcoind has not been compiled with zmq support.""" if not self.is_zmq_compiled(): @@ -856,6 +886,11 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): if not self.is_wallet_tool_compiled(): raise SkipTest("bitcoin-wallet has not been compiled") + def skip_if_no_bitcoin_util(self): + """Skip the running test if bitcoin-util has not been compiled.""" + if not self.is_bitcoin_util_compiled(): + raise SkipTest("bitcoin-util has not been compiled") + def skip_if_no_cli(self): """Skip the running test if bitcoin-cli has not been compiled.""" if not self.is_cli_compiled(): @@ -903,10 +938,18 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): """Checks whether bitcoin-wallet was compiled.""" return self.config["components"].getboolean("ENABLE_WALLET_TOOL") + def is_bitcoin_util_compiled(self): + """Checks whether bitcoin-util was compiled.""" + return self.config["components"].getboolean("ENABLE_BITCOIN_UTIL") + def is_zmq_compiled(self): """Checks whether the zmq module was compiled.""" return self.config["components"].getboolean("ENABLE_ZMQ") + def is_usdt_compiled(self): + """Checks whether the USDT tracepoints were compiled.""" + return self.config["components"].getboolean("ENABLE_USDT_TRACEPOINTS") + def is_sqlite_compiled(self): """Checks whether the wallet module was compiled with Sqlite support.""" return self.config["components"].getboolean("USE_SQLITE") diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index e56d4aa492..7d2db391b6 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -545,6 +545,7 @@ class TestNode(): Will throw if bitcoind starts without an error. Will throw if an expected_msg is provided and it does not match bitcoind's stdout.""" + assert not self.running with tempfile.NamedTemporaryFile(dir=self.stderr_dir, delete=False) as log_stderr, \ tempfile.NamedTemporaryFile(dir=self.stdout_dir, delete=False) as log_stdout: try: diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 210025104e..8651bcf636 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -573,17 +573,17 @@ def create_lots_of_big_transactions(node, txouts, utxos, num, fee): return txids -def mine_large_block(test_framework, node, utxos=None): +def mine_large_block(test_framework, mini_wallet, node): # generate a 66k transaction, # and 14 of them is close to the 1MB block limit - num = 14 txouts = gen_return_txouts() - utxos = utxos if utxos is not None else [] - if len(utxos) < num: - utxos.clear() - utxos.extend(node.listunspent()) - fee = 100 * node.getnetworkinfo()["relayfee"] - create_lots_of_big_transactions(node, txouts, utxos, num, fee=fee) + from .messages import COIN + fee = 100 * int(node.getnetworkinfo()["relayfee"] * COIN) + for _ in range(14): + tx = mini_wallet.create_self_transfer(from_node=node, fee_rate=0, mempool_valid=False)['tx'] + tx.vout[0].nValue -= fee + tx.vout.extend(txouts) + mini_wallet.sendrawtransaction(from_node=node, tx_hex=tx.serialize().hex()) test_framework.generate(node, 1) diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index dd41a740ae..6901bcfe66 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -8,7 +8,10 @@ from copy import deepcopy from decimal import Decimal from enum import Enum from random import choice -from typing import Optional +from typing import ( + Any, + Optional, +) from test_framework.address import ( base58_to_byte, create_deterministic_address_bcrt1_p2tr_op_true, @@ -93,6 +96,9 @@ class MiniWallet: self._address, self._internal_key = create_deterministic_address_bcrt1_p2tr_op_true() self._scriptPubKey = bytes.fromhex(self._test_node.validateaddress(self._address)['scriptPubKey']) + def get_balance(self): + return sum(u['value'] for u in self._utxos) + def rescan_utxos(self): """Drop all utxos and rescan the utxo set""" self._utxos = [] @@ -121,6 +127,7 @@ class MiniWallet: if not fixed_length: break tx.vin[0].scriptSig = CScript([der_sig + bytes(bytearray([SIGHASH_ALL]))]) + tx.rehash() def generate(self, num_blocks, **kwargs): """Generate blocks with coinbase outputs to the internal address, and append the outputs to the internal list""" @@ -131,24 +138,30 @@ class MiniWallet: self._utxos.append({'txid': cb_tx['txid'], 'vout': 0, 'value': cb_tx['vout'][0]['value'], 'height': block_info['height']}) return blocks + def get_scriptPubKey(self): + return self._scriptPubKey + def get_descriptor(self): return descsum_create(f'raw({self._scriptPubKey.hex()})') def get_address(self): return self._address - def get_utxo(self, *, txid: Optional[str]='', mark_as_spent=True): + def get_utxo(self, *, txid: str = '', vout: Optional[int] = None, mark_as_spent=True): """ Returns a utxo and marks it as spent (pops it from the internal list) Args: txid: get the first utxo we find from a specific transaction """ - index = -1 # by default the last utxo self._utxos = sorted(self._utxos, key=lambda k: (k['value'], -k['height'])) # Put the largest utxo last if txid: - utxo = next(filter(lambda utxo: txid == utxo['txid'], self._utxos)) - index = self._utxos.index(utxo) + utxo_filter: Any = filter(lambda utxo: txid == utxo['txid'], self._utxos) + else: + utxo_filter = reversed(self._utxos) # By default the largest utxo + if vout is not None: + utxo_filter = filter(lambda utxo: vout == utxo['vout'], utxo_filter) + index = self._utxos.index(next(utxo_filter)) if mark_as_spent: return self._utxos.pop(index) else: @@ -179,8 +192,50 @@ class MiniWallet: txid = self.sendrawtransaction(from_node=from_node, tx_hex=tx.serialize().hex()) return txid, 1 + def send_self_transfer_multi(self, **kwargs): + """ + Create and send a transaction that spends the given UTXOs and creates a + certain number of outputs with equal amounts. + + Returns a dictionary with + - txid + - serialized transaction in hex format + - transaction as CTransaction instance + - list of newly created UTXOs, ordered by vout index + """ + tx = self.create_self_transfer_multi(**kwargs) + txid = self.sendrawtransaction(from_node=kwargs['from_node'], tx_hex=tx.serialize().hex()) + return {'new_utxos': [self.get_utxo(txid=txid, vout=vout) for vout in range(len(tx.vout))], + 'txid': txid, 'hex': tx.serialize().hex(), 'tx': tx} + + def create_self_transfer_multi(self, *, from_node, utxos_to_spend=None, num_outputs=1, fee_per_output=1000): + """ + Create and return a transaction that spends the given UTXOs and creates a + certain number of outputs with equal amounts. + """ + utxos_to_spend = utxos_to_spend or [self.get_utxo()] + # create simple tx template (1 input, 1 output) + tx = self.create_self_transfer(fee_rate=0, from_node=from_node, utxo_to_spend=utxos_to_spend[0], mempool_valid=False)['tx'] + + # duplicate inputs, witnesses and outputs + tx.vin = [deepcopy(tx.vin[0]) for _ in range(len(utxos_to_spend))] + tx.wit.vtxinwit = [deepcopy(tx.wit.vtxinwit[0]) for _ in range(len(utxos_to_spend))] + tx.vout = [deepcopy(tx.vout[0]) for _ in range(num_outputs)] + + # adapt input prevouts + for i, utxo in enumerate(utxos_to_spend): + tx.vin[i] = CTxIn(COutPoint(int(utxo['txid'], 16), utxo['vout'])) + + # adapt output amounts (use fixed fee per output) + inputs_value_total = sum([int(COIN * utxo['value']) for utxo in utxos_to_spend]) + outputs_value_total = inputs_value_total - fee_per_output * num_outputs + for o in tx.vout: + o.nValue = outputs_value_total // num_outputs + return tx + def create_self_transfer(self, *, fee_rate=Decimal("0.003"), from_node=None, utxo_to_spend=None, mempool_valid=True, locktime=0, sequence=0): - """Create and return a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed.""" + """Create and return a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed. + Checking mempool validity via the testmempoolaccept RPC can be skipped by setting mempool_valid to False.""" from_node = from_node or self._test_node utxo_to_spend = utxo_to_spend or self.get_utxo() if self._priv_key is None: @@ -207,12 +262,13 @@ class MiniWallet: tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE]), bytes([LEAF_VERSION_TAPSCRIPT]) + self._internal_key] tx_hex = tx.serialize().hex() - tx_info = from_node.testmempoolaccept([tx_hex])[0] - assert_equal(mempool_valid, tx_info['allowed']) if mempool_valid: + tx_info = from_node.testmempoolaccept([tx_hex])[0] + assert_equal(tx_info['allowed'], True) assert_equal(tx_info['vsize'], vsize) assert_equal(tx_info['fees']['base'], utxo_to_spend['value'] - Decimal(send_value) / COIN) - return {'txid': tx_info['txid'], 'wtxid': tx_info['wtxid'], 'hex': tx_hex, 'tx': tx} + + return {'txid': tx.rehash(), 'wtxid': tx.getwtxid(), 'hex': tx_hex, 'tx': tx} def sendrawtransaction(self, *, from_node, tx_hex, **kwargs): txid = from_node.sendrawtransaction(hexstring=tx_hex, **kwargs) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index b0f24e3b97..8416a5881d 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -28,7 +28,7 @@ import logging import unittest # Formatting. Default colors to empty strings. -BOLD, GREEN, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "") +DEFAULT, BOLD, GREEN, RED = ("", ""), ("", ""), ("", ""), ("", "") try: # Make sure python thinks it can write unicode to its stdout "\u2713".encode("utf_8").decode(sys.stdout.encoding) @@ -59,10 +59,10 @@ if os.name != 'nt' or sys.getwindowsversion() >= (10, 0, 14393): #type:ignore kernel32.SetConsoleMode(stderr, stderr_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING) # primitive formatting on supported # terminal via ANSI escape sequences: + DEFAULT = ('\033[0m', '\033[0m') BOLD = ('\033[0m', '\033[1m') GREEN = ('\033[0m', '\033[0;32m') RED = ('\033[0m', '\033[0;31m') - GREY = ('\033[0m', '\033[1;30m') TEST_EXIT_PASSED = 0 TEST_EXIT_SKIPPED = 77 @@ -82,6 +82,7 @@ EXTENDED_SCRIPTS = [ # Longest test should go first, to favor running tests in parallel 'feature_pruning.py', 'feature_dbcrash.py', + 'feature_index_prune.py', ] BASE_SCRIPTS = [ @@ -111,7 +112,6 @@ BASE_SCRIPTS = [ 'p2p_tx_download.py', 'mempool_updatefromblock.py', 'wallet_dump.py --legacy-wallet', - 'feature_taproot.py --previous_release', 'feature_taproot.py', 'rpc_signer.py', 'wallet_signer.py --descriptors', @@ -145,6 +145,8 @@ BASE_SCRIPTS = [ 'wallet_txn_doublespend.py --mineblock', 'tool_wallet.py --legacy-wallet', 'tool_wallet.py --descriptors', + 'tool_signet_miner.py --legacy-wallet', + 'tool_signet_miner.py --descriptors', 'wallet_txn_clone.py', 'wallet_txn_clone.py --segwit', 'rpc_getchaintips.py', @@ -168,6 +170,10 @@ BASE_SCRIPTS = [ 'wallet_reorgsrestore.py', 'interface_http.py', 'interface_rpc.py', + 'interface_usdt_coinselection.py', + 'interface_usdt_net.py', + 'interface_usdt_utxocache.py', + 'interface_usdt_validation.py', 'rpc_psbt.py --legacy-wallet', 'rpc_psbt.py --descriptors', 'rpc_users.py', @@ -188,8 +194,7 @@ BASE_SCRIPTS = [ 'rpc_decodescript.py', 'rpc_blockchain.py', 'rpc_deprecated.py', - 'wallet_disable.py --legacy-wallet', - 'wallet_disable.py --descriptors', + 'wallet_disable.py', 'p2p_addr_relay.py', 'p2p_getaddr_caching.py', 'p2p_getdata.py', @@ -225,8 +230,7 @@ BASE_SCRIPTS = [ 'feature_rbf.py --descriptors', 'mempool_packages.py', 'mempool_package_onemore.py', - 'rpc_createmultisig.py --legacy-wallet', - 'rpc_createmultisig.py --descriptors', + 'rpc_createmultisig.py', 'rpc_packages.py', 'mempool_package_limits.py', 'feature_versionbits_warning.py', @@ -237,7 +241,6 @@ BASE_SCRIPTS = [ 'p2p_eviction.py', 'wallet_signmessagewithaddress.py', 'rpc_signmessagewithprivkey.py', - 'rpc_generateblock.py', 'rpc_generate.py', 'wallet_balance.py --legacy-wallet', 'wallet_balance.py --descriptors', @@ -252,6 +255,7 @@ BASE_SCRIPTS = [ 'rpc_bind.py --ipv4', 'rpc_bind.py --ipv6', 'rpc_bind.py --nonloopback', + 'wallet_crosschain.py', 'mining_basic.py', 'feature_signet.py', 'wallet_bumpfee.py --legacy-wallet', @@ -280,6 +284,8 @@ BASE_SCRIPTS = [ 'wallet_create_tx.py --legacy-wallet', 'wallet_send.py --legacy-wallet', 'wallet_send.py --descriptors', + 'wallet_sendall.py --legacy-wallet', + 'wallet_sendall.py --descriptors', 'wallet_create_tx.py --descriptors', 'wallet_taproot.py', 'wallet_inactive_hdchains.py', @@ -307,10 +313,10 @@ BASE_SCRIPTS = [ 'p2p_ping.py', 'rpc_scantxoutset.py', 'feature_txindex_compatibility.py', + 'feature_unsupported_utxo_db.py', 'feature_logging.py', 'feature_anchors.py', - 'feature_coinstatsindex.py --legacy-wallet', - 'feature_coinstatsindex.py --descriptors', + 'feature_coinstatsindex.py', 'wallet_orphanedreward.py', 'wallet_timelock.py', 'p2p_node_network_limited.py', @@ -328,7 +334,6 @@ BASE_SCRIPTS = [ 'feature_help.py', 'feature_shutdown.py', 'p2p_ibd_txrelay.py', - 'feature_blockfilterindex_prune.py' # Don't append tests at the end to avoid merge conflicts # Put them in a random line within the section that fits their approximate run-time ] @@ -367,11 +372,11 @@ def main(): args, unknown_args = parser.parse_known_args() if not args.ansi: - global BOLD, GREEN, RED, GREY + global DEFAULT, BOLD, GREEN, RED + DEFAULT = ("", "") BOLD = ("", "") GREEN = ("", "") RED = ("", "") - GREY = ("", "") # args to be passed on always start with two dashes; tests are the remaining unknown args tests = [arg for arg in unknown_args if arg[:2] != "--"] @@ -589,11 +594,12 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= # Clean up dangling processes if any. This may only happen with --failfast option. # Killing the process group will also terminate the current process but that is # not an issue - if len(job_queue.jobs): + if not os.getenv("CI_FAILFAST_TEST_LEAVE_DANGLING") and len(job_queue.jobs): os.killpg(os.getpgid(0), signal.SIGKILL) sys.exit(not all_passed) + def print_results(test_results, max_len_name, runtime): results = "\n" + BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "STATUS ", "DURATION") + BOLD[0] @@ -714,7 +720,7 @@ class TestResult(): color = RED glyph = CROSS elif self.status == "Skipped": - color = GREY + color = DEFAULT glyph = CIRCLE return color[1] + "%s | %s%s | %s s\n" % (self.name.ljust(self.padding), glyph, self.status.ljust(7), self.time) + color[0] diff --git a/test/functional/tool_signet_miner.py b/test/functional/tool_signet_miner.py new file mode 100755 index 0000000000..e6fc9072ab --- /dev/null +++ b/test/functional/tool_signet_miner.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test signet miner tool""" + +import os.path +import subprocess +import sys +import time + +from test_framework.key import ECKey +from test_framework.script_util import key_to_p2wpkh_script +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal +from test_framework.wallet_util import bytes_to_wif + + +CHALLENGE_PRIVATE_KEY = (42).to_bytes(32, 'big') + + +class SignetMinerTest(BitcoinTestFramework): + def set_test_params(self): + self.chain = "signet" + self.setup_clean_chain = True + self.num_nodes = 1 + + # generate and specify signet challenge (simple p2wpkh script) + privkey = ECKey() + privkey.set(CHALLENGE_PRIVATE_KEY, True) + pubkey = privkey.get_pubkey().get_bytes() + challenge = key_to_p2wpkh_script(pubkey) + self.extra_args = [[f'-signetchallenge={challenge.hex()}']] + + def skip_test_if_missing_module(self): + self.skip_if_no_cli() + self.skip_if_no_wallet() + self.skip_if_no_bitcoin_util() + + def run_test(self): + node = self.nodes[0] + # import private key needed for signing block + node.importprivkey(bytes_to_wif(CHALLENGE_PRIVATE_KEY)) + + # generate block with signet miner tool + base_dir = self.config["environment"]["SRCDIR"] + signet_miner_path = os.path.join(base_dir, "contrib", "signet", "miner") + subprocess.run([ + sys.executable, + signet_miner_path, + f'--cli={node.cli.binary} -datadir={node.cli.datadir}', + 'generate', + f'--address={node.getnewaddress()}', + f'--grind-cmd={self.options.bitcoinutil} grind', + '--nbits=1d00ffff', + f'--set-block-time={int(time.time())}', + ], check=True, stderr=subprocess.STDOUT) + assert_equal(node.getblockcount(), 1) + + +if __name__ == "__main__": + SignetMinerTest().main() diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py index dc823c2c60..f663666f57 100755 --- a/test/functional/wallet_avoidreuse.py +++ b/test/functional/wallet_avoidreuse.py @@ -118,6 +118,17 @@ class AvoidReuseTest(BitcoinTestFramework): assert_raises_rpc_error(-8, "Wallet flag is already set to false", self.nodes[0].setwalletflag, 'avoid_reuse', False) assert_raises_rpc_error(-8, "Wallet flag is already set to true", self.nodes[1].setwalletflag, 'avoid_reuse', True) + # Create a wallet with avoid reuse, and test that disabling it afterwards persists + self.nodes[1].createwallet(wallet_name="avoid_reuse_persist", avoid_reuse=True) + w = self.nodes[1].get_wallet_rpc("avoid_reuse_persist") + assert_equal(w.getwalletinfo()["avoid_reuse"], True) + w.setwalletflag("avoid_reuse", False) + assert_equal(w.getwalletinfo()["avoid_reuse"], False) + w.unloadwallet() + self.nodes[1].loadwallet("avoid_reuse_persist") + assert_equal(w.getwalletinfo()["avoid_reuse"], False) + w.unloadwallet() + def test_immutable(self): '''Test immutable wallet flags''' self.log.info("Test immutable wallet flags") diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index 0cfbefb719..0c93821e7f 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -50,7 +50,9 @@ class WalletTest(BitcoinTestFramework): self.num_nodes = 2 self.setup_clean_chain = True self.extra_args = [ - ['-limitdescendantcount=3'], # Limit mempool descendants as a hack to have wallet txs rejected from the mempool + # Limit mempool descendants as a hack to have wallet txs rejected from the mempool. + # Set walletrejectlongchains=0 so the wallet still creates the transactions. + ['-limitdescendantcount=3', '-walletrejectlongchains=0'], [], ] diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 69f9df57d8..a6c93ba5f9 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -25,7 +25,7 @@ class WalletTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 self.extra_args = [[ - "-acceptnonstdtxn=1", + "-acceptnonstdtxn=1", "-walletrejectlongchains=0" ]] * self.num_nodes self.setup_clean_chain = True self.supports_cli = False @@ -142,7 +142,7 @@ class WalletTest(BitcoinTestFramework): self.nodes[2].lockunspent(False, [unspent_0], True) # Restarting the node with the lock written to the wallet should keep the lock - self.restart_node(2) + self.restart_node(2, ["-walletrejectlongchains=0"]) assert_raises_rpc_error(-8, "Invalid parameter, output already locked", self.nodes[2].lockunspent, False, [unspent_0]) # Unloading and reloading the wallet with a persistent lock should keep the lock @@ -568,7 +568,7 @@ class WalletTest(BitcoinTestFramework): self.log.info("Test -reindex") self.stop_nodes() # set lower ancestor limit for later - self.start_node(0, ['-reindex', "-limitancestorcount=" + str(chainlimit)]) + self.start_node(0, ['-reindex', "-walletrejectlongchains=0", "-limitancestorcount=" + str(chainlimit)]) self.start_node(1, ['-reindex', "-limitancestorcount=" + str(chainlimit)]) self.start_node(2, ['-reindex', "-limitancestorcount=" + str(chainlimit)]) # reindex will leave rpc warm up "early"; Wait for it to finish @@ -668,7 +668,7 @@ class WalletTest(BitcoinTestFramework): "category": baz["category"], "vout": baz["vout"]} expected_fields = frozenset({'amount', 'bip125-replaceable', 'confirmations', 'details', 'fee', - 'hex', 'time', 'timereceived', 'trusted', 'txid', 'walletconflicts'}) + 'hex', 'time', 'timereceived', 'trusted', 'txid', 'wtxid', 'walletconflicts'}) verbose_field = "decoded" expected_verbose_fields = expected_fields | {verbose_field} diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index f6843d597d..3b23ee8e94 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -442,7 +442,9 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): self.generate(peer_node, 1) # Create single-input PSBT for transaction to be bumped - psbt = watcher.walletcreatefundedpsbt([], {dest_address: 0.0005}, 0, {"fee_rate": 1}, True)['psbt'] + # Ensure the payment amount + change can be fully funded using one of the 0.001BTC inputs. + psbt = watcher.walletcreatefundedpsbt([watcher.listunspent()[0]], {dest_address: 0.0005}, 0, + {"fee_rate": 1, "add_inputs": False}, True)['psbt'] psbt_signed = signer.walletprocesspsbt(psbt=psbt, sign=True, sighashtype="ALL", bip32derivs=True) psbt_final = watcher.finalizepsbt(psbt_signed["psbt"]) original_txid = watcher.sendrawtransaction(psbt_final["hex"]) diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py index 4416a9655f..12480d4d1e 100755 --- a/test/functional/wallet_createwallet.py +++ b/test/functional/wallet_createwallet.py @@ -26,6 +26,11 @@ class CreateWalletTest(BitcoinTestFramework): node = self.nodes[0] self.generate(node, 1) # Leave IBD for sethdseed + self.log.info("Run createwallet with invalid parameters.") + # Run createwallet with invalid parameters. This must not prevent a new wallet with the same name from being created with the correct parameters. + assert_raises_rpc_error(-4, "Passphrase provided but private keys are disabled. A passphrase is only used to encrypt private keys, so cannot be used for wallets with private keys disabled.", + self.nodes[0].createwallet, wallet_name='w0', disable_private_keys=True, passphrase="passphrase") + self.nodes[0].createwallet(wallet_name='w0') w0 = node.get_wallet_rpc('w0') address1 = w0.getnewaddress() @@ -164,5 +169,10 @@ class CreateWalletTest(BitcoinTestFramework): self.log.info('Using a passphrase with private keys disabled returns error') assert_raises_rpc_error(-4, 'Passphrase provided but private keys are disabled. A passphrase is only used to encrypt private keys, so cannot be used for wallets with private keys disabled.', self.nodes[0].createwallet, wallet_name='w9', disable_private_keys=True, passphrase='thisisapassphrase') + if self.is_bdb_compiled(): + self.log.info("Test legacy wallet deprecation") + res = self.nodes[0].createwallet(wallet_name="legacy_w0", descriptors=False, passphrase=None) + assert_equal(res["warning"], "Wallet created successfully. The legacy wallet type is being deprecated and support for creating and opening legacy wallets will be removed in the future.") + if __name__ == '__main__': CreateWalletTest().main() diff --git a/test/functional/wallet_crosschain.py b/test/functional/wallet_crosschain.py new file mode 100755 index 0000000000..b6d0c87985 --- /dev/null +++ b/test/functional/wallet_crosschain.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import os + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_raises_rpc_error + +class WalletCrossChain(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.setup_clean_chain = True + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def setup_network(self): + self.add_nodes(self.num_nodes) + + # Switch node 1 to testnet before starting it. + self.nodes[1].chain = 'testnet3' + self.nodes[1].extra_args = ['-maxconnections=0'] # disable testnet sync + with open(self.nodes[1].bitcoinconf, 'r', encoding='utf8') as conf: + conf_data = conf.read() + with open (self.nodes[1].bitcoinconf, 'w', encoding='utf8') as conf: + conf.write(conf_data.replace('regtest=', 'testnet=').replace('[regtest]', '[test]')) + + self.start_nodes() + + def run_test(self): + self.log.info("Creating wallets") + + node0_wallet = os.path.join(self.nodes[0].datadir, 'node0_wallet') + self.nodes[0].createwallet(node0_wallet) + self.nodes[0].unloadwallet(node0_wallet) + node1_wallet = os.path.join(self.nodes[1].datadir, 'node1_wallet') + self.nodes[1].createwallet(node1_wallet) + self.nodes[1].unloadwallet(node1_wallet) + + self.log.info("Loading wallets into nodes with a different genesis blocks") + + if self.options.descriptors: + assert_raises_rpc_error(-18, 'Wallet file verification failed.', self.nodes[0].loadwallet, node1_wallet) + assert_raises_rpc_error(-18, 'Wallet file verification failed.', self.nodes[1].loadwallet, node0_wallet) + else: + assert_raises_rpc_error(-4, 'Wallet files should not be reused across chains.', self.nodes[0].loadwallet, node1_wallet) + assert_raises_rpc_error(-4, 'Wallet files should not be reused across chains.', self.nodes[1].loadwallet, node0_wallet) + + if not self.options.descriptors: + self.log.info("Override cross-chain wallet load protection") + self.stop_nodes() + self.start_nodes([['-walletcrosschain']] * self.num_nodes) + self.nodes[0].loadwallet(node1_wallet) + self.nodes[1].loadwallet(node0_wallet) + + +if __name__ == '__main__': + WalletCrossChain().main() diff --git a/test/functional/wallet_disable.py b/test/functional/wallet_disable.py index 2c7996ca6b..74cddf2738 100755 --- a/test/functional/wallet_disable.py +++ b/test/functional/wallet_disable.py @@ -26,10 +26,6 @@ class DisableWalletTest (BitcoinTestFramework): x = self.nodes[0].validateaddress('mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ') assert x['isvalid'] == True - # Checking mining to an address without a wallet. Generating to a valid address should succeed - # but generating to an invalid address will fail. - self.generatetoaddress(self.nodes[0], 1, 'mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ') - assert_raises_rpc_error(-5, "Invalid address", self.generatetoaddress, self.nodes[0], 1, '3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy') if __name__ == '__main__': - DisableWalletTest ().main () + DisableWalletTest().main() diff --git a/test/functional/wallet_importprunedfunds.py b/test/functional/wallet_importprunedfunds.py index cdb5823109..2a4d0981c7 100755 --- a/test/functional/wallet_importprunedfunds.py +++ b/test/functional/wallet_importprunedfunds.py @@ -5,9 +5,13 @@ """Test the importprunedfunds and removeprunedfunds RPCs.""" from decimal import Decimal -from test_framework.blocktools import COINBASE_MATURITY from test_framework.address import key_to_p2wpkh +from test_framework.blocktools import COINBASE_MATURITY from test_framework.key import ECKey +from test_framework.messages import ( + CMerkleBlock, + from_hex, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -15,6 +19,7 @@ from test_framework.util import ( ) from test_framework.wallet_util import bytes_to_wif + class ImportPrunedFundsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -124,5 +129,18 @@ class ImportPrunedFundsTest(BitcoinTestFramework): w1.removeprunedfunds(txnid3) assert not [tx for tx in w1.listtransactions(include_watchonly=True) if tx['txid'] == txnid3] + # Check various RPC parameter validation errors + assert_raises_rpc_error(-22, "TX decode failed", w1.importprunedfunds, b'invalid tx'.hex(), proof1) + assert_raises_rpc_error(-5, "Transaction given doesn't exist in proof", w1.importprunedfunds, rawtxn2, proof1) + + mb = from_hex(CMerkleBlock(), proof1) + mb.header.hashMerkleRoot = 0xdeadbeef # cause mismatch between merkle root and merkle block + assert_raises_rpc_error(-5, "Something wrong with merkleblock", w1.importprunedfunds, rawtxn1, mb.serialize().hex()) + + mb = from_hex(CMerkleBlock(), proof1) + mb.header.nTime += 1 # modify arbitrary block header field to change block hash + assert_raises_rpc_error(-5, "Block not found in chain", w1.importprunedfunds, rawtxn1, mb.serialize().hex()) + + if __name__ == '__main__': ImportPrunedFundsTest().main() diff --git a/test/functional/wallet_listreceivedby.py b/test/functional/wallet_listreceivedby.py index 48b92796fc..a7f4f9ffaf 100755 --- a/test/functional/wallet_listreceivedby.py +++ b/test/functional/wallet_listreceivedby.py @@ -26,9 +26,6 @@ class ReceivedByTest(BitcoinTestFramework): self.skip_if_no_cli() def run_test(self): - # Generate block to get out of IBD - self.generate(self.nodes[0], 1) - # save the number of coinbase reward addresses so far num_cb_reward_addresses = len(self.nodes[1].listreceivedbyaddress(minconf=0, include_empty=True, include_watchonly=True)) @@ -172,7 +169,7 @@ class ReceivedByTest(BitcoinTestFramework): address = self.nodes[0].getnewaddress(label) reward = Decimal("25") - self.generatetoaddress(self.nodes[0], 1, address, sync_fun=self.no_op) + self.generatetoaddress(self.nodes[0], 1, address) hash = self.nodes[0].getbestblockhash() self.log.info("getreceivedbyaddress returns nothing with defaults") @@ -212,7 +209,7 @@ class ReceivedByTest(BitcoinTestFramework): {"label": label, "amount": reward}) self.log.info("Generate 100 more blocks") - self.generate(self.nodes[0], COINBASE_MATURITY, sync_fun=self.no_op) + self.generate(self.nodes[0], COINBASE_MATURITY) self.log.info("getreceivedbyaddress returns reward with defaults") balance = self.nodes[0].getreceivedbyaddress(address) diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index 5aae2c813a..6552bfe60c 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -38,7 +38,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): # Can take a few seconds due to transaction trickling peer_first.wait_for_broadcast([txid]) - # Add a second peer since txs aren't rebroadcast to the same peer (see filterInventoryKnown) + # Add a second peer since txs aren't rebroadcast to the same peer (see m_tx_inventory_known_filter) peer_second = node.add_p2p_connection(P2PTxInvStore()) self.log.info("Create a block") diff --git a/test/functional/wallet_send.py b/test/functional/wallet_send.py index 86e36be8f7..07baa0595e 100755 --- a/test/functional/wallet_send.py +++ b/test/functional/wallet_send.py @@ -10,6 +10,10 @@ from itertools import product from test_framework.authproxy import JSONRPCException from test_framework.descriptors import descsum_create from test_framework.key import ECKey +from test_framework.messages import ( + ser_compact_size, + WITNESS_SCALE_FACTOR, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -488,8 +492,8 @@ class WalletSendTest(BitcoinTestFramework): self.nodes[1].createwallet("extfund") ext_fund = self.nodes[1].get_wallet_rpc("extfund") - # Make a weird but signable script. sh(pkh()) descriptor accomplishes this - desc = descsum_create("sh(pkh({}))".format(privkey)) + # Make a weird but signable script. sh(wsh(pkh())) descriptor accomplishes this + desc = descsum_create("sh(wsh(pkh({})))".format(privkey)) if self.options.descriptors: res = ext_fund.importdescriptors([{"desc": desc, "timestamp": "now"}]) else: @@ -507,7 +511,7 @@ class WalletSendTest(BitcoinTestFramework): self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, expect_error=(-4, "Insufficient funds")) # But funding should work when the solving data is provided - res = self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, solving_data={"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}) + res = self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, solving_data={"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"], addr_info["embedded"]["embedded"]["scriptPubKey"]]}) signed = ext_wallet.walletprocesspsbt(res["psbt"]) signed = ext_fund.walletprocesspsbt(res["psbt"]) assert signed["complete"] @@ -526,10 +530,11 @@ class WalletSendTest(BitcoinTestFramework): break psbt_in = dec["inputs"][input_idx] # Calculate the input weight - # (prevout + sequence + length of scriptSig + 2 bytes buffer) * 4 + len of scriptwitness + # (prevout + sequence + length of scriptSig + scriptsig + 1 byte buffer) * WITNESS_SCALE_FACTOR + num scriptWitness stack items + (length of stack item + stack item) * N stack items + 1 byte buffer len_scriptsig = len(psbt_in["final_scriptSig"]["hex"]) // 2 if "final_scriptSig" in psbt_in else 0 - len_scriptwitness = len(psbt_in["final_scriptwitness"]["hex"]) // 2 if "final_scriptwitness" in psbt_in else 0 - input_weight = ((41 + len_scriptsig + 2) * 4) + len_scriptwitness + len_scriptsig += len(ser_compact_size(len_scriptsig)) + 1 + len_scriptwitness = (sum([(len(x) // 2) + len(ser_compact_size(len(x) // 2)) for x in psbt_in["final_scriptwitness"]]) + len(psbt_in["final_scriptwitness"]) + 1) if "final_scriptwitness" in psbt_in else 0 + input_weight = ((40 + len_scriptsig) * WITNESS_SCALE_FACTOR) + len_scriptwitness # Input weight error conditions assert_raises_rpc_error( diff --git a/test/functional/wallet_sendall.py b/test/functional/wallet_sendall.py new file mode 100755 index 0000000000..aa8d2a9d2c --- /dev/null +++ b/test/functional/wallet_sendall.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the sendall RPC command.""" + +from decimal import Decimal, getcontext + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_rpc_error, +) + +# Decorator to reset activewallet to zero utxos +def cleanup(func): + def wrapper(self): + try: + func(self) + finally: + if 0 < self.wallet.getbalances()["mine"]["trusted"]: + self.wallet.sendall([self.remainder_target]) + assert_equal(0, self.wallet.getbalances()["mine"]["trusted"]) # wallet is empty + return wrapper + +class SendallTest(BitcoinTestFramework): + # Setup and helpers + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def set_test_params(self): + getcontext().prec=10 + self.num_nodes = 1 + self.setup_clean_chain = True + + def assert_balance_swept_completely(self, tx, balance): + output_sum = sum([o["value"] for o in tx["decoded"]["vout"]]) + assert_equal(output_sum, balance + tx["fee"]) + assert_equal(0, self.wallet.getbalances()["mine"]["trusted"]) # wallet is empty + + def assert_tx_has_output(self, tx, addr, value=None): + for output in tx["decoded"]["vout"]: + if addr == output["scriptPubKey"]["address"] and value is None or value == output["value"]: + return + raise AssertionError("Output to {} not present or wrong amount".format(addr)) + + def assert_tx_has_outputs(self, tx, expected_outputs): + assert_equal(len(expected_outputs), len(tx["decoded"]["vout"])) + for eo in expected_outputs: + self.assert_tx_has_output(tx, eo["address"], eo["value"]) + + def add_utxos(self, amounts): + for a in amounts: + self.def_wallet.sendtoaddress(self.wallet.getnewaddress(), a) + self.generate(self.nodes[0], 1) + assert_greater_than(self.wallet.getbalances()["mine"]["trusted"], 0) + return self.wallet.getbalances()["mine"]["trusted"] + + # Helper schema for success cases + def test_sendall_success(self, sendall_args, remaining_balance = 0): + sendall_tx_receipt = self.wallet.sendall(sendall_args) + self.generate(self.nodes[0], 1) + # wallet has remaining balance (usually empty) + assert_equal(remaining_balance, self.wallet.getbalances()["mine"]["trusted"]) + + assert_equal(sendall_tx_receipt["complete"], True) + return self.wallet.gettransaction(txid = sendall_tx_receipt["txid"], verbose = True) + + @cleanup + def gen_and_clean(self): + self.add_utxos([15, 2, 4]) + + def test_cleanup(self): + self.log.info("Test that cleanup wrapper empties wallet") + self.gen_and_clean() + assert_equal(0, self.wallet.getbalances()["mine"]["trusted"]) # wallet is empty + + # Actual tests + @cleanup + def sendall_two_utxos(self): + self.log.info("Testing basic sendall case without specific amounts") + pre_sendall_balance = self.add_utxos([10,11]) + tx_from_wallet = self.test_sendall_success(sendall_args = [self.remainder_target]) + + self.assert_tx_has_outputs(tx = tx_from_wallet, + expected_outputs = [ + { "address": self.remainder_target, "value": pre_sendall_balance + tx_from_wallet["fee"] } # fee is neg + ] + ) + self.assert_balance_swept_completely(tx_from_wallet, pre_sendall_balance) + + @cleanup + def sendall_split(self): + self.log.info("Testing sendall where two recipients have unspecified amount") + pre_sendall_balance = self.add_utxos([1, 2, 3, 15]) + tx_from_wallet = self.test_sendall_success([self.remainder_target, self.split_target]) + + half = (pre_sendall_balance + tx_from_wallet["fee"]) / 2 + self.assert_tx_has_outputs(tx_from_wallet, + expected_outputs = [ + { "address": self.split_target, "value": half }, + { "address": self.remainder_target, "value": half } + ] + ) + self.assert_balance_swept_completely(tx_from_wallet, pre_sendall_balance) + + @cleanup + def sendall_and_spend(self): + self.log.info("Testing sendall in combination with paying specified amount to recipient") + pre_sendall_balance = self.add_utxos([8, 13]) + tx_from_wallet = self.test_sendall_success([{self.recipient: 5}, self.remainder_target]) + + self.assert_tx_has_outputs(tx_from_wallet, + expected_outputs = [ + { "address": self.recipient, "value": 5 }, + { "address": self.remainder_target, "value": pre_sendall_balance - 5 + tx_from_wallet["fee"] } + ] + ) + self.assert_balance_swept_completely(tx_from_wallet, pre_sendall_balance) + + @cleanup + def sendall_invalid_recipient_addresses(self): + self.log.info("Test having only recipient with specified amount, missing recipient with unspecified amount") + self.add_utxos([12, 9]) + + assert_raises_rpc_error( + -8, + "Must provide at least one address without a specified amount" , + self.wallet.sendall, + [{self.recipient: 5}] + ) + + @cleanup + def sendall_duplicate_recipient(self): + self.log.info("Test duplicate destination") + self.add_utxos([1, 8, 3, 9]) + + assert_raises_rpc_error( + -8, + "Invalid parameter, duplicated address: {}".format(self.remainder_target), + self.wallet.sendall, + [self.remainder_target, self.remainder_target] + ) + + @cleanup + def sendall_invalid_amounts(self): + self.log.info("Test sending more than balance") + pre_sendall_balance = self.add_utxos([7, 14]) + + expected_tx = self.wallet.sendall(recipients=[{self.recipient: 5}, self.remainder_target], options={"add_to_wallet": False}) + tx = self.wallet.decoderawtransaction(expected_tx['hex']) + fee = 21 - sum([o["value"] for o in tx["vout"]]) + + assert_raises_rpc_error(-6, "Assigned more value to outputs than available funds.", self.wallet.sendall, + [{self.recipient: pre_sendall_balance + 1}, self.remainder_target]) + assert_raises_rpc_error(-6, "Insufficient funds for fees after creating specified outputs.", self.wallet.sendall, + [{self.recipient: pre_sendall_balance}, self.remainder_target]) + assert_raises_rpc_error(-8, "Specified output amount to {} is below dust threshold".format(self.recipient), + self.wallet.sendall, [{self.recipient: 0.00000001}, self.remainder_target]) + assert_raises_rpc_error(-6, "Dynamically assigned remainder results in dust output.", self.wallet.sendall, + [{self.recipient: pre_sendall_balance - fee}, self.remainder_target]) + assert_raises_rpc_error(-6, "Dynamically assigned remainder results in dust output.", self.wallet.sendall, + [{self.recipient: pre_sendall_balance - fee - Decimal(0.00000010)}, self.remainder_target]) + + # @cleanup not needed because different wallet used + def sendall_negative_effective_value(self): + self.log.info("Test that sendall fails if all UTXOs have negative effective value") + # Use dedicated wallet for dust amounts and unload wallet at end + self.nodes[0].createwallet("dustwallet") + dust_wallet = self.nodes[0].get_wallet_rpc("dustwallet") + + self.def_wallet.sendtoaddress(dust_wallet.getnewaddress(), 0.00000400) + self.def_wallet.sendtoaddress(dust_wallet.getnewaddress(), 0.00000300) + self.generate(self.nodes[0], 1) + assert_greater_than(dust_wallet.getbalances()["mine"]["trusted"], 0) + + assert_raises_rpc_error(-6, "Total value of UTXO pool too low to pay for transaction." + + " Try using lower feerate or excluding uneconomic UTXOs with 'send_max' option.", + dust_wallet.sendall, recipients=[self.remainder_target], fee_rate=300) + + dust_wallet.unloadwallet() + + @cleanup + def sendall_with_send_max(self): + self.log.info("Check that `send_max` option causes negative value UTXOs to be left behind") + self.add_utxos([0.00000400, 0.00000300, 1]) + + # sendall with send_max + sendall_tx_receipt = self.wallet.sendall(recipients=[self.remainder_target], fee_rate=300, options={"send_max": True}) + tx_from_wallet = self.wallet.gettransaction(txid = sendall_tx_receipt["txid"], verbose = True) + + assert_equal(len(tx_from_wallet["decoded"]["vin"]), 1) + self.assert_tx_has_outputs(tx_from_wallet, [{"address": self.remainder_target, "value": 1 + tx_from_wallet["fee"]}]) + assert_equal(self.wallet.getbalances()["mine"]["trusted"], Decimal("0.00000700")) + + self.def_wallet.sendtoaddress(self.wallet.getnewaddress(), 1) + self.generate(self.nodes[0], 1) + + @cleanup + def sendall_specific_inputs(self): + self.log.info("Test sendall with a subset of UTXO pool") + self.add_utxos([17, 4]) + utxo = self.wallet.listunspent()[0] + + sendall_tx_receipt = self.wallet.sendall(recipients=[self.remainder_target], options={"inputs": [utxo]}) + tx_from_wallet = self.wallet.gettransaction(txid = sendall_tx_receipt["txid"], verbose = True) + assert_equal(len(tx_from_wallet["decoded"]["vin"]), 1) + assert_equal(len(tx_from_wallet["decoded"]["vout"]), 1) + assert_equal(tx_from_wallet["decoded"]["vin"][0]["txid"], utxo["txid"]) + assert_equal(tx_from_wallet["decoded"]["vin"][0]["vout"], utxo["vout"]) + self.assert_tx_has_output(tx_from_wallet, self.remainder_target) + + self.generate(self.nodes[0], 1) + assert_greater_than(self.wallet.getbalances()["mine"]["trusted"], 0) + + @cleanup + def sendall_fails_on_missing_input(self): + # fails because UTXO was previously spent, and wallet is empty + self.log.info("Test sendall fails because specified UTXO is not available") + self.add_utxos([16, 5]) + spent_utxo = self.wallet.listunspent()[0] + + # fails on unconfirmed spent UTXO + self.wallet.sendall(recipients=[self.remainder_target]) + assert_raises_rpc_error(-8, + "Input not available. UTXO ({}:{}) was already spent.".format(spent_utxo["txid"], spent_utxo["vout"]), + self.wallet.sendall, recipients=[self.remainder_target], options={"inputs": [spent_utxo]}) + + # fails on specific previously spent UTXO, while other UTXOs exist + self.generate(self.nodes[0], 1) + self.add_utxos([19, 2]) + assert_raises_rpc_error(-8, + "Input not available. UTXO ({}:{}) was already spent.".format(spent_utxo["txid"], spent_utxo["vout"]), + self.wallet.sendall, recipients=[self.remainder_target], options={"inputs": [spent_utxo]}) + + # fails because UTXO is unknown, while other UTXOs exist + foreign_utxo = self.def_wallet.listunspent()[0] + assert_raises_rpc_error(-8, "Input not found. UTXO ({}:{}) is not part of wallet.".format(foreign_utxo["txid"], + foreign_utxo["vout"]), self.wallet.sendall, recipients=[self.remainder_target], + options={"inputs": [foreign_utxo]}) + + @cleanup + def sendall_fails_on_no_address(self): + self.log.info("Test sendall fails because no address is provided") + self.add_utxos([19, 2]) + + assert_raises_rpc_error( + -8, + "Must provide at least one address without a specified amount" , + self.wallet.sendall, + [] + ) + + @cleanup + def sendall_fails_on_specific_inputs_with_send_max(self): + self.log.info("Test sendall fails because send_max is used while specific inputs are provided") + self.add_utxos([15, 6]) + utxo = self.wallet.listunspent()[0] + + assert_raises_rpc_error(-8, + "Cannot combine send_max with specific inputs.", + self.wallet.sendall, + recipients=[self.remainder_target], + options={"inputs": [utxo], "send_max": True}) + + def run_test(self): + self.nodes[0].createwallet("activewallet") + self.wallet = self.nodes[0].get_wallet_rpc("activewallet") + self.def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + self.generate(self.nodes[0], 101) + self.recipient = self.def_wallet.getnewaddress() # payee for a specific amount + self.remainder_target = self.def_wallet.getnewaddress() # address that receives everything left after payments and fees + self.split_target = self.def_wallet.getnewaddress() # 2nd target when splitting rest + + # Test cleanup + self.test_cleanup() + + # Basic sweep: everything to one address + self.sendall_two_utxos() + + # Split remainder to two addresses with equal amounts + self.sendall_split() + + # Pay recipient and sweep remainder + self.sendall_and_spend() + + # sendall fails if no recipient has unspecified amount + self.sendall_invalid_recipient_addresses() + + # Sendall fails if same destination is provided twice + self.sendall_duplicate_recipient() + + # Sendall fails when trying to spend more than the balance + self.sendall_invalid_amounts() + + # Sendall fails when wallet has no economically spendable UTXOs + self.sendall_negative_effective_value() + + # Leave dust behind if using send_max + self.sendall_with_send_max() + + # Sendall succeeds with specific inputs + self.sendall_specific_inputs() + + # Fails for the right reasons on missing or previously spent UTXOs + self.sendall_fails_on_missing_input() + + # Sendall fails when no address is provided + self.sendall_fails_on_no_address() + + # Sendall fails when using send_max while specifying inputs + self.sendall_fails_on_specific_inputs_with_send_max() + +if __name__ == '__main__': + SendallTest().main() diff --git a/test/functional/wallet_signer.py b/test/functional/wallet_signer.py index 423cfecdc0..8e4e1f5d36 100755 --- a/test/functional/wallet_signer.py +++ b/test/functional/wallet_signer.py @@ -194,6 +194,12 @@ class WalletSignerTest(BitcoinTestFramework): assert(res["complete"]) assert_equal(res["hex"], mock_tx) + self.log.info('Test sendall using hww1') + + res = hww.sendall(recipients=[{dest:0.5}, hww.getrawchangeaddress()],options={"add_to_wallet": False}) + assert(res["complete"]) + assert_equal(res["hex"], mock_tx) + # # Handle error thrown by script # self.set_mock_result(self.nodes[4], "2") # assert_raises_rpc_error(-1, 'Unable to parse JSON', diff --git a/test/functional/wallet_taproot.py b/test/functional/wallet_taproot.py index 17eab25457..41bb86f962 100755 --- a/test/functional/wallet_taproot.py +++ b/test/functional/wallet_taproot.py @@ -12,8 +12,11 @@ from test_framework.util import assert_equal from test_framework.descriptors import descsum_create from test_framework.script import ( CScript, + MAX_PUBKEYS_PER_MULTI_A, OP_1, OP_CHECKSIG, + OP_CHECKSIGADD, + OP_NUMEQUAL, taproot_construct, ) from test_framework.segwit_addr import encode_segwit_address @@ -167,6 +170,17 @@ def pk(hex_key): """Construct a script expression for taproot_construct for pk(hex_key).""" return (None, CScript([bytes.fromhex(hex_key), OP_CHECKSIG])) +def multi_a(k, hex_keys, sort=False): + """Construct a script expression for taproot_construct for a multi_a script.""" + xkeys = [bytes.fromhex(hex_key) for hex_key in hex_keys] + if sort: + xkeys.sort() + ops = [xkeys[0], OP_CHECKSIG] + for i in range(1, len(hex_keys)): + ops += [xkeys[i], OP_CHECKSIGADD] + ops += [k, OP_NUMEQUAL] + return (None, CScript(ops)) + def compute_taproot_address(pubkey, scripts): """Compute the address for a taproot output with given inner key and scripts.""" tap = taproot_construct(pubkey, scripts) @@ -178,9 +192,9 @@ class WalletTaprootTest(BitcoinTestFramework): """Test generation and spending of P2TR address outputs.""" def set_test_params(self): - self.num_nodes = 3 + self.num_nodes = 2 self.setup_clean_chain = True - self.extra_args = [['-keypool=100'], ['-keypool=100'], ["-vbparams=taproot:1:1"]] + self.extra_args = [['-keypool=100'], ['-keypool=100']] self.supports_cli = False def skip_test_if_missing_module(self): @@ -194,19 +208,6 @@ class WalletTaprootTest(BitcoinTestFramework): pass @staticmethod - def rand_keys(n): - ret = [] - idxes = set() - for _ in range(n): - while True: - i = random.randrange(len(KEYS)) - if not i in idxes: - break - idxes.add(i) - ret.append(KEYS[i]) - return ret - - @staticmethod def make_desc(pattern, privmap, keys, pub_only = False): pat = pattern.replace("$H", H_POINT) for i in range(len(privmap)): @@ -242,15 +243,11 @@ class WalletTaprootTest(BitcoinTestFramework): assert_equal(len(rederive), 1) assert_equal(rederive[0], addr_g) - # tr descriptors can be imported regardless of Taproot status + # tr descriptors can be imported result = self.privs_tr_enabled.importdescriptors([{"desc": desc, "timestamp": "now"}]) assert(result[0]["success"]) result = self.pubs_tr_enabled.importdescriptors([{"desc": desc_pub, "timestamp": "now"}]) assert(result[0]["success"]) - result = self.privs_tr_disabled.importdescriptors([{"desc": desc, "timestamp": "now"}]) - assert result[0]["success"] - result = self.pubs_tr_disabled.importdescriptors([{"desc": desc_pub, "timestamp": "now"}]) - assert result[0]["success"] def do_test_sendtoaddress(self, comment, pattern, privmap, treefn, keys_pay, keys_change): self.log.info("Testing %s through sendtoaddress" % comment) @@ -275,7 +272,8 @@ class WalletTaprootTest(BitcoinTestFramework): self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress(), sync_fun=self.no_op) test_balance = int(self.rpc_online.getbalance() * 100000000) ret_amnt = random.randrange(100000, test_balance) - res = self.rpc_online.sendtoaddress(address=self.boring.getnewaddress(), amount=Decimal(ret_amnt) / 100000000, subtractfeefromamount=True) + # Increase fee_rate to compensate for the wallet's inability to estimate fees for script path spends. + res = self.rpc_online.sendtoaddress(address=self.boring.getnewaddress(), amount=Decimal(ret_amnt) / 100000000, subtractfeefromamount=True, fee_rate=200) self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress(), sync_fun=self.no_op) assert(self.rpc_online.gettransaction(res)["confirmations"] > 0) @@ -306,7 +304,8 @@ class WalletTaprootTest(BitcoinTestFramework): self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress(), sync_fun=self.no_op) test_balance = int(self.psbt_online.getbalance() * 100000000) ret_amnt = random.randrange(100000, test_balance) - psbt = self.psbt_online.walletcreatefundedpsbt([], [{self.boring.getnewaddress(): Decimal(ret_amnt) / 100000000}], None, {"subtractFeeFromOutputs":[0]})['psbt'] + # Increase fee_rate to compensate for the wallet's inability to estimate fees for script path spends. + psbt = self.psbt_online.walletcreatefundedpsbt([], [{self.boring.getnewaddress(): Decimal(ret_amnt) / 100000000}], None, {"subtractFeeFromOutputs":[0], "fee_rate": 200})['psbt'] res = self.psbt_offline.walletprocesspsbt(psbt) assert(res['complete']) rawtx = self.nodes[0].finalizepsbt(res['psbt'])['hex'] @@ -314,8 +313,9 @@ class WalletTaprootTest(BitcoinTestFramework): self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress(), sync_fun=self.no_op) assert(self.psbt_online.gettransaction(txid)['confirmations'] > 0) - def do_test(self, comment, pattern, privmap, treefn, nkeys): - keys = self.rand_keys(nkeys * 4) + def do_test(self, comment, pattern, privmap, treefn): + nkeys = len(privmap) + keys = random.sample(KEYS, nkeys * 4) self.do_test_addr(comment, pattern, privmap, treefn, keys[0:nkeys]) self.do_test_sendtoaddress(comment, pattern, privmap, treefn, keys[0:nkeys], keys[nkeys:2*nkeys]) self.do_test_psbt(comment, pattern, privmap, treefn, keys[2*nkeys:3*nkeys], keys[3*nkeys:4*nkeys]) @@ -324,12 +324,8 @@ class WalletTaprootTest(BitcoinTestFramework): self.log.info("Creating wallets...") self.nodes[0].createwallet(wallet_name="privs_tr_enabled", descriptors=True, blank=True) self.privs_tr_enabled = self.nodes[0].get_wallet_rpc("privs_tr_enabled") - self.nodes[2].createwallet(wallet_name="privs_tr_disabled", descriptors=True, blank=True) - self.privs_tr_disabled=self.nodes[2].get_wallet_rpc("privs_tr_disabled") self.nodes[0].createwallet(wallet_name="pubs_tr_enabled", descriptors=True, blank=True, disable_private_keys=True) self.pubs_tr_enabled = self.nodes[0].get_wallet_rpc("pubs_tr_enabled") - self.nodes[2].createwallet(wallet_name="pubs_tr_disabled", descriptors=True, blank=True, disable_private_keys=True) - self.pubs_tr_disabled=self.nodes[2].get_wallet_rpc("pubs_tr_disabled") self.nodes[0].createwallet(wallet_name="boring") self.nodes[0].createwallet(wallet_name="addr_gen", descriptors=True, disable_private_keys=True, blank=True) self.nodes[0].createwallet(wallet_name="rpc_online", descriptors=True, blank=True) @@ -349,64 +345,98 @@ class WalletTaprootTest(BitcoinTestFramework): "tr(XPRV)", "tr($1/*)", [True], - lambda k1: (key(k1), []), - 1 + lambda k1: (key(k1), []) ) self.do_test( "tr(H,XPRV)", "tr($H,pk($1/*))", [True], - lambda k1: (key(H_POINT), [pk(k1)]), - 1 + lambda k1: (key(H_POINT), [pk(k1)]) ) self.do_test( "wpkh(XPRV)", "wpkh($1/*)", [True], - None, - 1 + None ) self.do_test( "tr(XPRV,{H,{H,XPUB}})", "tr($1/*,{pk($H),{pk($H),pk($2/*)}})", [True, False], - lambda k1, k2: (key(k1), [pk(H_POINT), [pk(H_POINT), pk(k2)]]), - 2 + lambda k1, k2: (key(k1), [pk(H_POINT), [pk(H_POINT), pk(k2)]]) ) self.do_test( "wsh(multi(1,XPRV,XPUB))", "wsh(multi(1,$1/*,$2/*))", [True, False], - None, - 2 + None ) self.do_test( "tr(XPRV,{XPUB,XPUB})", "tr($1/*,{pk($2/*),pk($2/*)})", [True, False], - lambda k1, k2: (key(k1), [pk(k2), pk(k2)]), - 2 + lambda k1, k2: (key(k1), [pk(k2), pk(k2)]) ) self.do_test( "tr(XPRV,{{XPUB,H},{H,XPUB}})", "tr($1/*,{{pk($2/*),pk($H)},{pk($H),pk($2/*)}})", [True, False], - lambda k1, k2: (key(k1), [[pk(k2), pk(H_POINT)], [pk(H_POINT), pk(k2)]]), - 2 + lambda k1, k2: (key(k1), [[pk(k2), pk(H_POINT)], [pk(H_POINT), pk(k2)]]) ) self.do_test( "tr(XPUB,{{H,{H,XPUB}},{H,{H,{H,XPRV}}}})", "tr($1/*,{{pk($H),{pk($H),pk($2/*)}},{pk($H),{pk($H),{pk($H),pk($3/*)}}}})", [False, False, True], - lambda k1, k2, k3: (key(k1), [[pk(H_POINT), [pk(H_POINT), pk(k2)]], [pk(H_POINT), [pk(H_POINT), [pk(H_POINT), pk(k3)]]]]), - 3 + lambda k1, k2, k3: (key(k1), [[pk(H_POINT), [pk(H_POINT), pk(k2)]], [pk(H_POINT), [pk(H_POINT), [pk(H_POINT), pk(k3)]]]]) ) self.do_test( "tr(XPRV,{XPUB,{{XPUB,{H,H}},{{H,H},XPUB}}})", "tr($1/*,{pk($2/*),{{pk($2/*),{pk($H),pk($H)}},{{pk($H),pk($H)},pk($2/*)}}})", [True, False], - lambda k1, k2: (key(k1), [pk(k2), [[pk(k2), [pk(H_POINT), pk(H_POINT)]], [[pk(H_POINT), pk(H_POINT)], pk(k2)]]]), - 2 + lambda k1, k2: (key(k1), [pk(k2), [[pk(k2), [pk(H_POINT), pk(H_POINT)]], [[pk(H_POINT), pk(H_POINT)], pk(k2)]]]) + ) + self.do_test( + "tr(H,multi_a(1,XPRV))", + "tr($H,multi_a(1,$1/*))", + [True], + lambda k1: (key(H_POINT), [multi_a(1, [k1])]) + ) + self.do_test( + "tr(H,sortedmulti_a(1,XPRV,XPUB))", + "tr($H,sortedmulti_a(1,$1/*,$2/*))", + [True, False], + lambda k1, k2: (key(H_POINT), [multi_a(1, [k1, k2], True)]) + ) + self.do_test( + "tr(H,{H,multi_a(1,XPUB,XPRV)})", + "tr($H,{pk($H),multi_a(1,$1/*,$2/*)})", + [False, True], + lambda k1, k2: (key(H_POINT), [pk(H_POINT), [multi_a(1, [k1, k2])]]) + ) + self.do_test( + "tr(H,sortedmulti_a(1,XPUB,XPRV,XPRV))", + "tr($H,sortedmulti_a(1,$1/*,$2/*,$3/*))", + [False, True, True], + lambda k1, k2, k3: (key(H_POINT), [multi_a(1, [k1, k2, k3], True)]) + ) + self.do_test( + "tr(H,multi_a(2,XPRV,XPUB,XPRV))", + "tr($H,multi_a(2,$1/*,$2/*,$3/*))", + [True, False, True], + lambda k1, k2, k3: (key(H_POINT), [multi_a(2, [k1, k2, k3])]) + ) + self.do_test( + "tr(XPUB,{{XPUB,{XPUB,sortedmulti_a(2,XPRV,XPUB,XPRV)}})", + "tr($2/*,{pk($2/*),{pk($2/*),sortedmulti_a(2,$1/*,$2/*,$3/*)}})", + [True, False, True], + lambda k1, k2, k3: (key(k2), [pk(k2), [pk(k2), multi_a(2, [k1, k2, k3], True)]]) + ) + rnd_pos = random.randrange(MAX_PUBKEYS_PER_MULTI_A) + self.do_test( + "tr(XPUB,multi_a(1,H...,XPRV,H...))", + "tr($2/*,multi_a(1" + (",$H" * rnd_pos) + ",$1/*" + (",$H" * (MAX_PUBKEYS_PER_MULTI_A - 1 - rnd_pos)) + "))", + [True, False], + lambda k1, k2: (key(k2), [multi_a(1, ([H_POINT] * rnd_pos) + [k1] + ([H_POINT] * (MAX_PUBKEYS_PER_MULTI_A - 1 - rnd_pos)))]) ) self.log.info("Sending everything back...") diff --git a/test/get_previous_releases.py b/test/get_previous_releases.py index e0d48f8047..cbdb67216c 100755 --- a/test/get_previous_releases.py +++ b/test/get_previous_releases.py @@ -19,8 +19,12 @@ import subprocess import sys import hashlib - SHA256_SUMS = { + "0e2819135366f150d9906e294b61dff58fd1996ebd26c2f8e979d6c0b7a79580": "bitcoin-0.14.3-aarch64-linux-gnu.tar.gz", + "d86fc90824a85c38b25c8488115178d5785dbc975f5ff674f9f5716bc8ad6e65": "bitcoin-0.14.3-arm-linux-gnueabihf.tar.gz", + "1b0a7408c050e3d09a8be8e21e183ef7ee570385dc41216698cc3ab392a484e7": "bitcoin-0.14.3-osx64.tar.gz", + "706e0472dbc933ed2757650d54cbcd780fd3829ebf8f609b32780c7eedebdbc9": "bitcoin-0.14.3-x86_64-linux-gnu.tar.gz", + # "d40f18b4e43c6e6370ef7db9131f584fbb137276ec2e3dba67a4b267f81cb644": "bitcoin-0.15.2-aarch64-linux-gnu.tar.gz", "54fb877a148a6ad189a1e1ab1ff8b11181e58ff2aaf430da55b3fd46ae549a6b": "bitcoin-0.15.2-arm-linux-gnueabihf.tar.gz", "87e9340ff3d382d543b2b69112376077f0c8b4f7450d372e83b68f5a1e22b2df": "bitcoin-0.15.2-osx64.tar.gz", @@ -67,6 +71,15 @@ SHA256_SUMS = { "91b1e012975c5a363b5b5fcc81b5b7495e86ff703ec8262d4b9afcfec633c30d": "bitcoin-22.0-powerpc64le-linux-gnu.tar.gz", "9cc3a62c469fe57e11485fdd32c916f10ce7a2899299855a2e479256ff49ff3c": "bitcoin-22.0-riscv64-linux-gnu.tar.gz", "59ebd25dd82a51638b7a6bb914586201e67db67b919b2a1ff08925a7936d1b16": "bitcoin-22.0-x86_64-linux-gnu.tar.gz", + + "06f4c78271a77752ba5990d60d81b1751507f77efda1e5981b4e92fd4d9969fb": "bitcoin-23.0-aarch64-linux-gnu.tar.gz", + "952c574366aff76f6d6ad1c9ee45a361d64fa04155e973e926dfe7e26f9703a3": "bitcoin-23.0-arm-linux-gnueabihf.tar.gz", + "7c8bc63731aa872b7b334a8a7d96e33536ad77d49029bad179b09dca32cd77ac": "bitcoin-23.0-arm64-apple-darwin.tar.gz", + "2caa5898399e415f61d9af80a366a3008e5856efa15aaff74b88acf429674c99": "bitcoin-23.0-powerpc64-linux-gnu.tar.gz", + "217dd0469d0f4962d22818c368358575f6a0abcba8804807bb75325eb2f28b19": "bitcoin-23.0-powerpc64le-linux-gnu.tar.gz", + "078f96b1e92895009c798ab827fb3fde5f6719eee886bd0c0e93acab18ea4865": "bitcoin-23.0-riscv64-linux-gnu.tar.gz", + "c816780583009a9dad426dc0c183c89be9da98906e1e2c7ebae91041c1aaaaf3": "bitcoin-23.0-x86_64-apple-darwin.tar.gz", + "2cca490c1f2842884a3c5b0606f179f9f937177da4eadd628e3f7fd7e25d26d0": "bitcoin-23.0-x86_64-linux-gnu.tar.gz", } @@ -92,8 +105,11 @@ def download_binary(tag, args) -> int: if match: bin_path = 'bin/bitcoin-core-{}/test.{}'.format( match.group(1), match.group(2)) + platform = args.platform + if tag < "v23" and platform in ["x86_64-apple-darwin", "aarch64-apple-darwin"]: + platform = "osx64" tarball = 'bitcoin-{tag}-{platform}.tar.gz'.format( - tag=tag[1:], platform=args.platform) + tag=tag[1:], platform=platform) tarballUrl = 'https://bitcoincore.org/{bin_path}/{tarball}'.format( bin_path=bin_path, tarball=tarball) @@ -197,8 +213,8 @@ def check_host(args) -> int: platforms = { 'aarch64-*-linux*': 'aarch64-linux-gnu', 'x86_64-*-linux*': 'x86_64-linux-gnu', - 'x86_64-apple-darwin*': 'osx64', - 'aarch64-apple-darwin*': 'osx64', + 'x86_64-apple-darwin*': 'x86_64-apple-darwin', + 'aarch64-apple-darwin*': 'aarch64-apple-darwin', } args.platform = '' for pattern, target in platforms.items(): diff --git a/test/lint/README.md b/test/lint/README.md index f4165f908e..1f683c10b3 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -39,6 +39,6 @@ To do so, add the upstream repository as remote: git remote add --fetch secp256k1 https://github.com/bitcoin-core/secp256k1.git ``` -lint-all.sh +lint-all.py =========== Calls other scripts with the `lint-` prefix. diff --git a/test/lint/extended-lint-all.sh b/test/lint/extended-lint-all.sh deleted file mode 100755 index be5d9db4a9..0000000000 --- a/test/lint/extended-lint-all.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# This script runs all contrib/devtools/extended-lint-*.sh files, and fails if -# any exit with a non-zero status code. - -# This script is intentionally locale dependent by not setting "export LC_ALL=C" -# in order to allow for the executed lint scripts to opt in or opt out of locale -# dependence themselves. - -set -u - -SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") -LINTALL=$(basename "${BASH_SOURCE[0]}") - -for f in "${SCRIPTDIR}"/extended-lint-*.sh; do - if [ "$(basename "$f")" != "$LINTALL" ]; then - if ! "$f"; then - echo "^---- failure generated from $f" - exit 1 - fi - fi -done diff --git a/test/lint/extended-lint-cppcheck.sh b/test/lint/extended-lint-cppcheck.sh deleted file mode 100755 index 2af39ed60a..0000000000 --- a/test/lint/extended-lint-cppcheck.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# - -export LC_ALL=C - -ENABLED_CHECKS=( - "Class '.*' has a constructor with 1 argument that is not explicit." - "Struct '.*' has a constructor with 1 argument that is not explicit." -) - -IGNORED_WARNINGS=( - "src/arith_uint256.h:.* Class 'arith_uint256' has a constructor with 1 argument that is not explicit." - "src/arith_uint256.h:.* Class 'base_uint < 256 >' has a constructor with 1 argument that is not explicit." - "src/arith_uint256.h:.* Class 'base_uint' has a constructor with 1 argument that is not explicit." - "src/coins.h:.* Class 'CCoinsViewBacked' has a constructor with 1 argument that is not explicit." - "src/coins.h:.* Class 'CCoinsViewCache' has a constructor with 1 argument that is not explicit." - "src/coins.h:.* Class 'CCoinsViewCursor' has a constructor with 1 argument that is not explicit." - "src/net.h:.* Class 'CNetMessage' has a constructor with 1 argument that is not explicit." - "src/policy/feerate.h:.* Class 'CFeeRate' has a constructor with 1 argument that is not explicit." - "src/prevector.h:.* Class 'const_iterator' has a constructor with 1 argument that is not explicit." - "src/prevector.h:.* Class 'const_reverse_iterator' has a constructor with 1 argument that is not explicit." - "src/prevector.h:.* Class 'iterator' has a constructor with 1 argument that is not explicit." - "src/prevector.h:.* Class 'reverse_iterator' has a constructor with 1 argument that is not explicit." - "src/primitives/block.h:.* Class 'CBlock' has a constructor with 1 argument that is not explicit." - "src/primitives/transaction.h:.* Class 'CTransaction' has a constructor with 1 argument that is not explicit." - "src/protocol.h:.* Class 'CMessageHeader' has a constructor with 1 argument that is not explicit." - "src/qt/guiutil.h:.* Class 'ItemDelegate' has a constructor with 1 argument that is not explicit." - "src/rpc/util.h:.* Struct 'RPCResults' has a constructor with 1 argument that is not explicit." - "src/rpc/util.h:.* Struct 'UniValueType' has a constructor with 1 argument that is not explicit." - "src/rpc/util.h:.* style: Struct 'UniValueType' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'AddressDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'ComboDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'ConstPubkeyProvider' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'PKDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'PKHDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'RawDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'SHDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'WPKHDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'WSHDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/script.h:.* Class 'CScript' has a constructor with 1 argument that is not explicit." - "src/script/standard.h:.* Class 'CScriptID' has a constructor with 1 argument that is not explicit." - "src/span.h:.* Class 'Span < const CRPCCommand >' has a constructor with 1 argument that is not explicit." - "src/span.h:.* Class 'Span < const char >' has a constructor with 1 argument that is not explicit." - "src/span.h:.* Class 'Span < const std :: vector <unsigned char > >' has a constructor with 1 argument that is not explicit." - "src/span.h:.* Class 'Span < const uint8_t >' has a constructor with 1 argument that is not explicit." - "src/span.h:.* Class 'Span' has a constructor with 1 argument that is not explicit." - "src/support/allocators/secure.h:.* Struct 'secure_allocator < char >' has a constructor with 1 argument that is not explicit." - "src/support/allocators/secure.h:.* Struct 'secure_allocator < RNGState >' has a constructor with 1 argument that is not explicit." - "src/support/allocators/secure.h:.* Struct 'secure_allocator < unsigned char >' has a constructor with 1 argument that is not explicit." - "src/support/allocators/zeroafterfree.h:.* Struct 'zero_after_free_allocator < char >' has a constructor with 1 argument that is not explicit." - "src/test/checkqueue_tests.cpp:.* Struct 'FailingCheck' has a constructor with 1 argument that is not explicit." - "src/test/checkqueue_tests.cpp:.* Struct 'MemoryCheck' has a constructor with 1 argument that is not explicit." - "src/test/checkqueue_tests.cpp:.* Struct 'UniqueCheck' has a constructor with 1 argument that is not explicit." - "src/test/fuzz/util.h:.* Class 'FuzzedFileProvider' has a constructor with 1 argument that is not explicit." - "src/test/fuzz/util.h:.* Class 'FuzzedAutoFileProvider' has a constructor with 1 argument that is not explicit." - "src/wallet/db.h:.* Class 'BerkeleyEnvironment' has a constructor with 1 argument that is not explicit." -) - -if ! command -v cppcheck > /dev/null; then - echo "Skipping cppcheck linting since cppcheck is not installed. Install by running \"apt install cppcheck\"" - exit 0 -fi - -function join_array { - local IFS="$1" - shift - echo "$*" -} - -ENABLED_CHECKS_REGEXP=$(join_array "|" "${ENABLED_CHECKS[@]}") -IGNORED_WARNINGS_REGEXP=$(join_array "|" "${IGNORED_WARNINGS[@]}") -WARNINGS=$(git ls-files -- "*.cpp" "*.h" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" | \ - xargs cppcheck --enable=all -j "$(getconf _NPROCESSORS_ONLN)" --language=c++ --std=c++17 --template=gcc -D__cplusplus -DCLIENT_VERSION_BUILD -DCLIENT_VERSION_IS_RELEASE -DCLIENT_VERSION_MAJOR -DCLIENT_VERSION_MINOR -DCOPYRIGHT_YEAR -DDEBUG -I src/ -q 2>&1 | sort -u | \ - grep -E "${ENABLED_CHECKS_REGEXP}" | \ - grep -vE "${IGNORED_WARNINGS_REGEXP}") -if [[ ${WARNINGS} != "" ]]; then - echo "${WARNINGS}" - echo - echo "Advice not applicable in this specific case? Add an exception by updating" - echo "IGNORED_WARNINGS in $0" - # Uncomment to enforce the developer note policy "By default, declare single-argument constructors `explicit`" - # exit 1 -fi -exit 0 diff --git a/test/lint/lint-all.py b/test/lint/lint-all.py new file mode 100755 index 0000000000..c280ba2db2 --- /dev/null +++ b/test/lint/lint-all.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2017-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# This script runs all test/lint/lint-* files, and fails if any exit +# with a non-zero status code. + +from glob import glob +from pathlib import Path +from subprocess import run + +exit_code = 0 +mod_path = Path(__file__).parent +for lint in glob(f"{mod_path}/lint-*"): + if lint != __file__: + result = run([lint]) + if result.returncode != 0: + print(f"^---- failure generated from {lint.split('/')[-1]}") + exit_code |= result.returncode + +exit(exit_code) diff --git a/test/lint/lint-all.sh b/test/lint/lint-all.sh deleted file mode 100755 index fabc24c91b..0000000000 --- a/test/lint/lint-all.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2017-2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# This script runs all contrib/devtools/lint-*.sh files, and fails if any exit -# with a non-zero status code. - -# This script is intentionally locale dependent by not setting "export LC_ALL=C" -# in order to allow for the executed lint scripts to opt in or opt out of locale -# dependence themselves. - -set -u - -SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") -LINTALL=$(basename "${BASH_SOURCE[0]}") - -EXIT_CODE=0 - -for f in "${SCRIPTDIR}"/lint-*.sh; do - if [ "$(basename "$f")" != "$LINTALL" ]; then - if ! "$f"; then - echo "^---- failure generated from $f" - EXIT_CODE=1 - fi - fi -done - -exit ${EXIT_CODE} diff --git a/test/lint/lint-assertions.py b/test/lint/lint-assertions.py new file mode 100755 index 0000000000..195ff33d11 --- /dev/null +++ b/test/lint/lint-assertions.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for assertions with obvious side effects. + +import sys +import subprocess + + +def git_grep(params: [], error_msg: ""): + try: + output = subprocess.check_output(["git", "grep", *params], universal_newlines=True, encoding="utf8") + print(error_msg) + print(output) + return 1 + except subprocess.CalledProcessError as ex1: + if ex1.returncode > 1: + raise ex1 + return 0 + + +def main(): + # PRE31-C (SEI CERT C Coding Standard): + # "Assertions should not contain assignments, increment, or decrement operators." + exit_code = git_grep([ + "-E", + r"[^_]assert\(.*(\+\+|\-\-|[^=!<>]=[^=!<>]).*\);", + "--", + "*.cpp", + "*.h", + ], "Assertions should not have side effects:") + + # Aborting the whole process is undesirable for RPC code. So nonfatal + # checks should be used over assert. See: src/util/check.h + # src/rpc/server.cpp is excluded from this check since it's mostly meta-code. + exit_code |= git_grep([ + "-nE", + r"\<(A|a)ss(ume|ert) *\(.*\);", + "--", + "src/rpc/", + "src/wallet/rpc*", + ":(exclude)src/rpc/server.cpp", + ], "CHECK_NONFATAL(condition) or NONFATAL_UNREACHABLE should be used instead of assert for RPC code.") + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-assertions.sh b/test/lint/lint-assertions.sh deleted file mode 100755 index 2860f5621b..0000000000 --- a/test/lint/lint-assertions.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for assertions with obvious side effects. - -export LC_ALL=C - -EXIT_CODE=0 - -# PRE31-C (SEI CERT C Coding Standard): -# "Assertions should not contain assignments, increment, or decrement operators." -OUTPUT=$(git grep -E '[^_]assert\(.*(\+\+|\-\-|[^=!<>]=[^=!<>]).*\);' -- "*.cpp" "*.h") -if [[ ${OUTPUT} != "" ]]; then - echo "Assertions should not have side effects:" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi - -# Macro CHECK_NONFATAL(condition) should be used instead of assert for RPC code, where it -# is undesirable to crash the whole program. See: src/util/check.h -# src/rpc/server.cpp is excluded from this check since it's mostly meta-code. -OUTPUT=$(git grep -nE '\<(A|a)ssert *\(.*\);' -- "src/rpc/" "src/wallet/rpc*" ":(exclude)src/rpc/server.cpp") -if [[ ${OUTPUT} != "" ]]; then - echo "CHECK_NONFATAL(condition) should be used instead of assert for RPC code." - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi - -exit ${EXIT_CODE} diff --git a/test/lint/lint-circular-dependencies.py b/test/lint/lint-circular-dependencies.py new file mode 100755 index 0000000000..7ca2ec994b --- /dev/null +++ b/test/lint/lint-circular-dependencies.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2020-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for circular dependencies + +import os +import re +import subprocess +import sys + +EXPECTED_CIRCULAR_DEPENDENCIES = ( + "chainparamsbase -> util/system -> chainparamsbase", + "node/blockstorage -> validation -> node/blockstorage", + "index/coinstatsindex -> node/coinstats -> index/coinstatsindex", + "policy/fees -> txmempool -> policy/fees", + "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel", + "qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel", + "qt/sendcoinsdialog -> qt/walletmodel -> qt/sendcoinsdialog", + "qt/transactiontablemodel -> qt/walletmodel -> qt/transactiontablemodel", + "wallet/fees -> wallet/wallet -> wallet/fees", + "wallet/wallet -> wallet/walletdb -> wallet/wallet", + "node/coinstats -> validation -> node/coinstats", +) + +CODE_DIR = "src" + + +def main(): + circular_dependencies = [] + exit_code = 0 + + os.chdir(CODE_DIR) + files = subprocess.check_output( + ['git', 'ls-files', '--', '*.h', '*.cpp'], + universal_newlines=True, + ).splitlines() + + command = [sys.executable, "../contrib/devtools/circular-dependencies.py", *files] + dependencies_output = subprocess.run( + command, + stdout=subprocess.PIPE, + universal_newlines=True, + ) + + for dependency_str in dependencies_output.stdout.rstrip().split("\n"): + circular_dependencies.append( + re.sub("^Circular dependency: ", "", dependency_str) + ) + + # Check for an unexpected dependencies + for dependency in circular_dependencies: + if dependency not in EXPECTED_CIRCULAR_DEPENDENCIES: + exit_code = 1 + print( + f'A new circular dependency in the form of "{dependency}" appears to have been introduced.\n', + file=sys.stderr, + ) + + # Check for missing expected dependencies + for expected_dependency in EXPECTED_CIRCULAR_DEPENDENCIES: + if expected_dependency not in circular_dependencies: + exit_code = 1 + print( + f'Good job! The circular dependency "{expected_dependency}" is no longer present.', + ) + print( + f"Please remove it from EXPECTED_CIRCULAR_DEPENDENCIES in {__file__}", + ) + print( + "to make sure this circular dependency is not accidentally reintroduced.\n", + ) + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh deleted file mode 100755 index 69185090d1..0000000000 --- a/test/lint/lint-circular-dependencies.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for circular dependencies - -export LC_ALL=C - -EXPECTED_CIRCULAR_DEPENDENCIES=( - "chainparamsbase -> util/system -> chainparamsbase" - "node/blockstorage -> validation -> node/blockstorage" - "index/blockfilterindex -> node/blockstorage -> validation -> index/blockfilterindex" - "index/base -> validation -> index/blockfilterindex -> index/base" - "index/coinstatsindex -> node/coinstats -> index/coinstatsindex" - "policy/fees -> txmempool -> policy/fees" - "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel" - "qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel" - "qt/sendcoinsdialog -> qt/walletmodel -> qt/sendcoinsdialog" - "qt/transactiontablemodel -> qt/walletmodel -> qt/transactiontablemodel" - "wallet/fees -> wallet/wallet -> wallet/fees" - "wallet/wallet -> wallet/walletdb -> wallet/wallet" - "node/coinstats -> validation -> node/coinstats" -) - -EXIT_CODE=0 - -CIRCULAR_DEPENDENCIES=() - -IFS=$'\n' -for CIRC in $(cd src && ../contrib/devtools/circular-dependencies.py {*,*/*,*/*/*}.{h,cpp} | sed -e 's/^Circular dependency: //'); do - CIRCULAR_DEPENDENCIES+=( "$CIRC" ) - IS_EXPECTED_CIRC=0 - for EXPECTED_CIRC in "${EXPECTED_CIRCULAR_DEPENDENCIES[@]}"; do - if [[ "${CIRC}" == "${EXPECTED_CIRC}" ]]; then - IS_EXPECTED_CIRC=1 - break - fi - done - if [[ ${IS_EXPECTED_CIRC} == 0 ]]; then - echo "A new circular dependency in the form of \"${CIRC}\" appears to have been introduced." - echo - EXIT_CODE=1 - fi -done - -for EXPECTED_CIRC in "${EXPECTED_CIRCULAR_DEPENDENCIES[@]}"; do - IS_PRESENT_EXPECTED_CIRC=0 - for CIRC in "${CIRCULAR_DEPENDENCIES[@]}"; do - if [[ "${CIRC}" == "${EXPECTED_CIRC}" ]]; then - IS_PRESENT_EXPECTED_CIRC=1 - break - fi - done - if [[ ${IS_PRESENT_EXPECTED_CIRC} == 0 ]]; then - echo "Good job! The circular dependency \"${EXPECTED_CIRC}\" is no longer present." - echo "Please remove it from EXPECTED_CIRCULAR_DEPENDENCIES in $0" - echo "to make sure this circular dependency is not accidentally reintroduced." - echo - EXIT_CODE=1 - fi -done - -exit ${EXIT_CODE} diff --git a/test/lint/lint-cpp.sh b/test/lint/lint-cpp.sh deleted file mode 100755 index cac57b968d..0000000000 --- a/test/lint/lint-cpp.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for various C++ code patterns we want to avoid. - -export LC_ALL=C - -EXIT_CODE=0 - -OUTPUT=$(git grep -E "boost::bind\(" -- "*.cpp" "*.h") -if [[ ${OUTPUT} != "" ]]; then - echo "Use of boost::bind detected. Use std::bind instead." - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi - -exit ${EXIT_CODE}
\ No newline at end of file diff --git a/test/lint/lint-files.py b/test/lint/lint-files.py index 68b795eef7..3723e0ee6a 100755 --- a/test/lint/lint-files.py +++ b/test/lint/lint-files.py @@ -13,6 +13,7 @@ import sys from subprocess import check_output from typing import Optional, NoReturn +CMD_TOP_LEVEL = ["git", "rev-parse", "--show-toplevel"] CMD_ALL_FILES = "git ls-files -z --full-name" CMD_SOURCE_FILES = 'git ls-files -z --full-name -- "*.[cC][pP][pP]" "*.[hH]" "*.[pP][yY]" "*.[sS][hH]"' CMD_SHEBANG_FILES = "git grep --full-name --line-number -I '^#!'" @@ -184,6 +185,8 @@ def check_shebang_file_permissions() -> int: def main() -> NoReturn: + root_dir = check_output(CMD_TOP_LEVEL).decode("utf8").strip() + os.chdir(root_dir) failed_tests = 0 failed_tests += check_all_filenames() failed_tests += check_source_filenames() diff --git a/test/lint/lint-files.sh b/test/lint/lint-files.sh deleted file mode 100755 index 86d7fc724a..0000000000 --- a/test/lint/lint-files.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C - -set -e -cd "$(dirname "$0")/../.." -test/lint/lint-files.py diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py index b814446125..28e7b1e4ff 100755 --- a/test/lint/lint-format-strings.py +++ b/test/lint/lint-format-strings.py @@ -1,291 +1,98 @@ #!/usr/bin/env python3 # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # -# Lint format strings: This program checks that the number of arguments passed -# to a variadic format string function matches the number of format specifiers -# in the format string. -import argparse +""" +Lint format strings: This program checks that the number of arguments passed +to a variadic format string function matches the number of format specifiers +in the format string. +""" + +import subprocess import re import sys -FALSE_POSITIVES = [ - ("src/dbwrapper.cpp", "vsnprintf(p, limit - p, format, backup_ap)"), - ("src/index/base.cpp", "FatalError(const char* fmt, const Args&... args)"), - ("src/netbase.cpp", "LogConnectFailure(bool manual_connection, const char* fmt, const Args&... args)"), - ("src/clientversion.cpp", "strprintf(_(COPYRIGHT_HOLDERS).translated, COPYRIGHT_HOLDERS_SUBSTITUTION)"), - ("src/validationinterface.cpp", "LogPrint(BCLog::VALIDATION, fmt \"\\n\", __VA_ARGS__)"), - ("src/wallet/wallet.h", "WalletLogPrintf(std::string fmt, Params... parameters)"), - ("src/wallet/wallet.h", "LogPrintf((\"%s \" + fmt).c_str(), GetDisplayName(), parameters...)"), - ("src/wallet/scriptpubkeyman.h", "WalletLogPrintf(std::string fmt, Params... parameters)"), - ("src/wallet/scriptpubkeyman.h", "LogPrintf((\"%s \" + fmt).c_str(), m_storage.GetDisplayName(), parameters...)"), +FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS = [ + 'FatalError,0', + 'fprintf,1', + 'tfm::format,1', # Assuming tfm::::format(std::ostream&, ... + 'LogConnectFailure,1', + 'LogPrint,1', + 'LogPrintf,0', + 'printf,0', + 'snprintf,2', + 'sprintf,1', + 'strprintf,0', + 'vfprintf,1', + 'vprintf,1', + 'vsnprintf,1', + 'vsprintf,1', + 'WalletLogPrintf,0', ] - - -def parse_function_calls(function_name, source_code): - """Return an array with all calls to function function_name in string source_code. - Preprocessor directives and C++ style comments ("//") in source_code are removed. - - >>> len(parse_function_calls("foo", "foo();bar();foo();bar();")) - 2 - >>> parse_function_calls("foo", "foo(1);bar(1);foo(2);bar(2);")[0].startswith("foo(1);") - True - >>> parse_function_calls("foo", "foo(1);bar(1);foo(2);bar(2);")[1].startswith("foo(2);") - True - >>> len(parse_function_calls("foo", "foo();bar();// foo();bar();")) - 1 - >>> len(parse_function_calls("foo", "#define FOO foo();")) - 0 - """ - assert type(function_name) is str and type(source_code) is str and function_name - lines = [re.sub("// .*", " ", line).strip() - for line in source_code.split("\n") - if not line.strip().startswith("#")] - return re.findall(r"[^a-zA-Z_](?=({}\(.*).*)".format(function_name), " " + " ".join(lines)) - - -def normalize(s): - """Return a normalized version of string s with newlines, tabs and C style comments ("/* ... */") - replaced with spaces. Multiple spaces are replaced with a single space. - - >>> normalize(" /* nothing */ foo\tfoo /* bar */ foo ") - 'foo foo foo' - """ - assert type(s) is str - s = s.replace("\n", " ") - s = s.replace("\t", " ") - s = re.sub(r"/\*.*?\*/", " ", s) - s = re.sub(" {2,}", " ", s) - return s.strip() - - -ESCAPE_MAP = { - r"\n": "[escaped-newline]", - r"\t": "[escaped-tab]", - r'\"': "[escaped-quote]", -} - - -def escape(s): - """Return the escaped version of string s with "\\\"", "\\n" and "\\t" escaped as - "[escaped-backslash]", "[escaped-newline]" and "[escaped-tab]". - - >>> unescape(escape("foo")) == "foo" - True - >>> escape(r'foo \\t foo \\n foo \\\\ foo \\ foo \\"bar\\"') - 'foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]' - """ - assert type(s) is str - for raw_value, escaped_value in ESCAPE_MAP.items(): - s = s.replace(raw_value, escaped_value) - return s - - -def unescape(s): - """Return the unescaped version of escaped string s. - Reverses the replacements made in function escape(s). - - >>> unescape(escape("bar")) - 'bar' - >>> unescape("foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]") - 'foo \\\\t foo \\\\n foo \\\\\\\\ foo \\\\ foo \\\\"bar\\\\"' - """ - assert type(s) is str - for raw_value, escaped_value in ESCAPE_MAP.items(): - s = s.replace(escaped_value, raw_value) - return s - - -def parse_function_call_and_arguments(function_name, function_call): - """Split string function_call into an array of strings consisting of: - * the string function_call followed by "(" - * the function call argument #1 - * ... - * the function call argument #n - * a trailing ");" - - The strings returned are in escaped form. See escape(...). - - >>> parse_function_call_and_arguments("foo", 'foo("%s", "foo");') - ['foo(', '"%s",', ' "foo"', ')'] - >>> parse_function_call_and_arguments("foo", 'foo("%s", "foo");') - ['foo(', '"%s",', ' "foo"', ')'] - >>> parse_function_call_and_arguments("foo", 'foo("%s %s", "foo", "bar");') - ['foo(', '"%s %s",', ' "foo",', ' "bar"', ')'] - >>> parse_function_call_and_arguments("fooprintf", 'fooprintf("%050d", i);') - ['fooprintf(', '"%050d",', ' i', ')'] - >>> parse_function_call_and_arguments("foo", 'foo(bar(foobar(barfoo("foo"))), foobar); barfoo') - ['foo(', 'bar(foobar(barfoo("foo"))),', ' foobar', ')'] - >>> parse_function_call_and_arguments("foo", "foo()") - ['foo(', '', ')'] - >>> parse_function_call_and_arguments("foo", "foo(123)") - ['foo(', '123', ')'] - >>> parse_function_call_and_arguments("foo", 'foo("foo")') - ['foo(', '"foo"', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>,wchar_t>().to_bytes(buf), err);') - ['strprintf(', '"%s (%d)",', ' std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>,wchar_t>().to_bytes(buf),', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo<wchar_t>().to_bytes(buf), err);') - ['strprintf(', '"%s (%d)",', ' foo<wchar_t>().to_bytes(buf),', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo().to_bytes(buf), err);') - ['strprintf(', '"%s (%d)",', ' foo().to_bytes(buf),', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo << 1, err);') - ['strprintf(', '"%s (%d)",', ' foo << 1,', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo<bar>() >> 1, err);') - ['strprintf(', '"%s (%d)",', ' foo<bar>() >> 1,', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo < 1 ? bar : foobar, err);') - ['strprintf(', '"%s (%d)",', ' foo < 1 ? bar : foobar,', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo < 1, err);') - ['strprintf(', '"%s (%d)",', ' foo < 1,', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo > 1 ? bar : foobar, err);') - ['strprintf(', '"%s (%d)",', ' foo > 1 ? bar : foobar,', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo > 1, err);') - ['strprintf(', '"%s (%d)",', ' foo > 1,', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo <= 1, err);') - ['strprintf(', '"%s (%d)",', ' foo <= 1,', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo <= bar<1, 2>(1, 2), err);') - ['strprintf(', '"%s (%d)",', ' foo <= bar<1, 2>(1, 2),', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo>foo<1,2>(1,2)?bar:foobar,err)'); - ['strprintf(', '"%s (%d)",', ' foo>foo<1,2>(1,2)?bar:foobar,', 'err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo>foo<1,2>(1,2),err)'); - ['strprintf(', '"%s (%d)",', ' foo>foo<1,2>(1,2),', 'err', ')'] - """ - assert type(function_name) is str and type(function_call) is str and function_name - remaining = normalize(escape(function_call)) - expected_function_call = "{}(".format(function_name) - assert remaining.startswith(expected_function_call) - parts = [expected_function_call] - remaining = remaining[len(expected_function_call):] - open_parentheses = 1 - open_template_arguments = 0 - in_string = False - parts.append("") - for i, char in enumerate(remaining): - parts.append(parts.pop() + char) - if char == "\"": - in_string = not in_string - continue - if in_string: - continue - if char == "(": - open_parentheses += 1 - continue - if char == ")": - open_parentheses -= 1 - if open_parentheses > 1: - continue - if open_parentheses == 0: - parts.append(parts.pop()[:-1]) - parts.append(char) - break - prev_char = remaining[i - 1] if i - 1 >= 0 else None - next_char = remaining[i + 1] if i + 1 <= len(remaining) - 1 else None - if char == "<" and next_char not in [" ", "<", "="] and prev_char not in [" ", "<"]: - open_template_arguments += 1 - continue - if char == ">" and next_char not in [" ", ">", "="] and prev_char not in [" ", ">"] and open_template_arguments > 0: - open_template_arguments -= 1 - if open_template_arguments > 0: - continue - if char == ",": - parts.append("") - return parts - - -def parse_string_content(argument): - """Return the text within quotes in string argument. - - >>> parse_string_content('1 "foo %d bar" 2') - 'foo %d bar' - >>> parse_string_content('1 foobar 2') - '' - >>> parse_string_content('1 "bar" 2') - 'bar' - >>> parse_string_content('1 "foo" 2 "bar" 3') - 'foobar' - >>> parse_string_content('1 "foo" 2 " " "bar" 3') - 'foo bar' - >>> parse_string_content('""') - '' - >>> parse_string_content('') - '' - >>> parse_string_content('1 2 3') - '' - """ - assert type(argument) is str - string_content = "" - in_string = False - for char in normalize(escape(argument)): - if char == "\"": - in_string = not in_string - elif in_string: - string_content += char - return string_content - - -def count_format_specifiers(format_string): - """Return the number of format specifiers in string format_string. - - >>> count_format_specifiers("foo bar foo") - 0 - >>> count_format_specifiers("foo %d bar foo") - 1 - >>> count_format_specifiers("foo %d bar %i foo") - 2 - >>> count_format_specifiers("foo %d bar %i foo %% foo") - 2 - >>> count_format_specifiers("foo %d bar %i foo %% foo %d foo") - 3 - >>> count_format_specifiers("foo %d bar %i foo %% foo %*d foo") - 4 - """ - assert type(format_string) is str - format_string = format_string.replace('%%', 'X') - n = 0 - in_specifier = False - for i, char in enumerate(format_string): - if char == "%": - in_specifier = True - n += 1 - elif char in "aAcdeEfFgGinopsuxX": - in_specifier = False - elif in_specifier and char == "*": - n += 1 - return n - +RUN_LINT_FILE = 'test/lint/run-lint-format-strings.py' + +def check_doctest(): + command = [ + sys.executable, + '-m', + 'doctest', + RUN_LINT_FILE, + ] + try: + subprocess.run(command, check = True) + except subprocess.CalledProcessError: + sys.exit(1) + +def get_matching_files(function_name): + command = [ + 'git', + 'grep', + '--full-name', + '-l', + function_name, + '--', + '*.c', + '*.cpp', + '*.h', + ] + try: + return subprocess.check_output(command, stderr = subprocess.STDOUT).decode('utf-8').splitlines() + except subprocess.CalledProcessError as e: + if e.returncode > 1: # return code is 1 when match is empty + print(e.output.decode('utf-8'), end='') + sys.exit(1) + return [] def main(): - parser = argparse.ArgumentParser(description="This program checks that the number of arguments passed " - "to a variadic format string function matches the number of format " - "specifiers in the format string.") - parser.add_argument("--skip-arguments", type=int, help="number of arguments before the format string " - "argument (e.g. 1 in the case of fprintf)", default=0) - parser.add_argument("function_name", help="function name (e.g. fprintf)", default=None) - parser.add_argument("file", nargs="*", help="C++ source code file (e.g. foo.cpp)") - args = parser.parse_args() exit_code = 0 - for filename in args.file: - with open(filename, "r", encoding="utf-8") as f: - for function_call_str in parse_function_calls(args.function_name, f.read()): - parts = parse_function_call_and_arguments(args.function_name, function_call_str) - relevant_function_call_str = unescape("".join(parts))[:512] - if (f.name, relevant_function_call_str) in FALSE_POSITIVES: - continue - if len(parts) < 3 + args.skip_arguments: - exit_code = 1 - print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str)) - continue - argument_count = len(parts) - 3 - args.skip_arguments - format_str = parse_string_content(parts[1 + args.skip_arguments]) - format_specifier_count = count_format_specifiers(format_str) - if format_specifier_count != argument_count: - exit_code = 1 - print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str)) - continue - sys.exit(exit_code) + check_doctest() + for s in FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS: + function_name, skip_arguments = s.split(',') + matching_files = get_matching_files(function_name) + + matching_files_filtered = [] + for matching_file in matching_files: + if not re.search('^src/(leveldb|secp256k1|minisketch|tinyformat|univalue|test/fuzz/strprintf.cpp)', matching_file): + matching_files_filtered.append(matching_file) + matching_files_filtered.sort() + + run_lint_args = [ + RUN_LINT_FILE, + '--skip-arguments', + skip_arguments, + function_name, + ] + run_lint_args.extend(matching_files_filtered) + + try: + subprocess.run(run_lint_args, check = True) + except subprocess.CalledProcessError: + exit_code = 1 + sys.exit(exit_code) -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/test/lint/lint-format-strings.sh b/test/lint/lint-format-strings.sh deleted file mode 100755 index d98f12b1a1..0000000000 --- a/test/lint/lint-format-strings.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Lint format strings: This program checks that the number of arguments passed -# to a variadic format string function matches the number of format specifiers -# in the format string. - -export LC_ALL=C - -FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS=( - "FatalError,0" - "fprintf,1" - "tfm::format,1" # Assuming tfm::::format(std::ostream&, ... - "LogConnectFailure,1" - "LogPrint,1" - "LogPrintf,0" - "printf,0" - "snprintf,2" - "sprintf,1" - "strprintf,0" - "vfprintf,1" - "vprintf,1" - "vsnprintf,1" - "vsprintf,1" - "WalletLogPrintf,0" -) - -EXIT_CODE=0 -if ! python3 -m doctest test/lint/lint-format-strings.py; then - EXIT_CODE=1 -fi -for S in "${FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS[@]}"; do - IFS="," read -r FUNCTION_NAME SKIP_ARGUMENTS <<< "${S}" - for MATCHING_FILE in $(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(leveldb|secp256k1|minisketch|tinyformat|univalue|test/fuzz/strprintf.cpp)"); do - MATCHING_FILES+=("${MATCHING_FILE}") - done - if ! test/lint/lint-format-strings.py --skip-arguments "${SKIP_ARGUMENTS}" "${FUNCTION_NAME}" "${MATCHING_FILES[@]}"; then - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-git-commit-check.py b/test/lint/lint-git-commit-check.py new file mode 100755 index 0000000000..a1d03370e8 --- /dev/null +++ b/test/lint/lint-git-commit-check.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2020-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Linter to check that commit messages have a new line before the body +# or no body at all + +import argparse +import os +import sys + +from subprocess import check_output + + +def parse_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description=""" + Linter to check that commit messages have a new line before + the body or no body at all. + """, + epilog=f""" + You can manually set the commit-range with the COMMIT_RANGE + environment variable (e.g. "COMMIT_RANGE='47ba2c3...ee50c9e' + {sys.argv[0]}"). Defaults to current merge base when neither + prev-commits nor the environment variable is set. + """) + + parser.add_argument("--prev-commits", "-p", required=False, help="The previous n commits to check") + + return parser.parse_args() + + +def main(): + args = parse_args() + exit_code = 0 + + if not os.getenv("COMMIT_RANGE"): + if args.prev_commits: + commit_range = "HEAD~" + args.prev_commits + "...HEAD" + else: + # This assumes that the target branch of the pull request will be master. + merge_base = check_output(["git", "merge-base", "HEAD", "master"], universal_newlines=True, encoding="utf8").rstrip("\n") + commit_range = merge_base + "..HEAD" + else: + commit_range = os.getenv("COMMIT_RANGE") + + commit_hashes = check_output(["git", "log", commit_range, "--format=%H"], universal_newlines=True, encoding="utf8").splitlines() + + for hash in commit_hashes: + commit_info = check_output(["git", "log", "--format=%B", "-n", "1", hash], universal_newlines=True, encoding="utf8").splitlines() + if len(commit_info) >= 2: + if commit_info[1]: + print(f"The subject line of commit hash {hash} is followed by a non-empty line. Subject lines should always be followed by a blank line.") + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-git-commit-check.sh b/test/lint/lint-git-commit-check.sh deleted file mode 100755 index f77373ed00..0000000000 --- a/test/lint/lint-git-commit-check.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2020-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Linter to check that commit messages have a new line before the body -# or no body at all - -export LC_ALL=C - -EXIT_CODE=0 - -while getopts "?" opt; do - case $opt in - ?) - echo "Usage: $0 [N]" - echo " COMMIT_RANGE='<commit range>' $0" - echo " $0 -?" - echo "Checks unmerged commits, the previous N commits, or a commit range." - echo "COMMIT_RANGE='47ba2c3...ee50c9e' $0" - exit ${EXIT_CODE} - ;; - esac -done - -if [ -z "${COMMIT_RANGE}" ]; then - if [ -n "$1" ]; then - COMMIT_RANGE="HEAD~$1...HEAD" - else - # This assumes that the target branch of the pull request will be master. - MERGE_BASE=$(git merge-base HEAD master) - COMMIT_RANGE="$MERGE_BASE..HEAD" - fi -fi - -while IFS= read -r commit_hash || [[ -n "$commit_hash" ]]; do - n_line=0 - while IFS= read -r line || [[ -n "$line" ]]; do - n_line=$((n_line+1)) - length=${#line} - if [ $n_line -eq 2 ] && [ "$length" -ne 0 ]; then - echo "The subject line of commit hash ${commit_hash} is followed by a non-empty line. Subject lines should always be followed by a blank line." - EXIT_CODE=1 - fi - done < <(git log --format=%B -n 1 "$commit_hash") -done < <(git log "${COMMIT_RANGE}" --format=%H) - -exit ${EXIT_CODE} diff --git a/test/lint/lint-include-guards.py b/test/lint/lint-include-guards.py new file mode 100755 index 0000000000..86284517d5 --- /dev/null +++ b/test/lint/lint-include-guards.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check include guards. +""" + +import re +import sys +from subprocess import check_output +from typing import List + + +HEADER_ID_PREFIX = 'BITCOIN_' +HEADER_ID_SUFFIX = '_H' + +EXCLUDE_FILES_WITH_PREFIX = ['src/crypto/ctaes', + 'src/leveldb', + 'src/crc32c', + 'src/secp256k1', + 'src/minisketch', + 'src/univalue', + 'src/tinyformat.h', + 'src/bench/nanobench.h', + 'src/test/fuzz/FuzzedDataProvider.h'] + + +def _get_header_file_lst() -> List[str]: + """ Helper function to get a list of header filepaths to be + checked for include guards. + """ + git_cmd_lst = ['git', 'ls-files', '--', '*.h'] + header_file_lst = check_output( + git_cmd_lst).decode('utf-8').splitlines() + + header_file_lst = [hf for hf in header_file_lst + if not any(ef in hf for ef + in EXCLUDE_FILES_WITH_PREFIX)] + + return header_file_lst + + +def _get_header_id(header_file: str) -> str: + """ Helper function to get the header id from a header file + string. + + eg: 'src/wallet/walletdb.h' -> 'BITCOIN_WALLET_WALLETDB_H' + + Args: + header_file: Filepath to header file. + + Returns: + The header id. + """ + header_id_base = header_file.split('/')[1:] + header_id_base = '_'.join(header_id_base) + header_id_base = header_id_base.replace('.h', '').replace('-', '_') + header_id_base = header_id_base.upper() + + header_id = f'{HEADER_ID_PREFIX}{header_id_base}{HEADER_ID_SUFFIX}' + + return header_id + + +def main(): + exit_code = 0 + + header_file_lst = _get_header_file_lst() + for header_file in header_file_lst: + header_id = _get_header_id(header_file) + + regex_pattern = f'^#(ifndef|define|endif //) {header_id}' + + with open(header_file, 'r', encoding='utf-8') as f: + header_file_contents = f.readlines() + + count = 0 + for header_file_contents_string in header_file_contents: + include_guard_lst = re.findall( + regex_pattern, header_file_contents_string) + + count += len(include_guard_lst) + + if count != 3: + print(f'{header_file} seems to be missing the expected ' + 'include guard:') + print(f' #ifndef {header_id}') + print(f' #define {header_id}') + print(' ...') + print(f' #endif // {header_id}\n') + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-include-guards.sh b/test/lint/lint-include-guards.sh deleted file mode 100755 index f14218aa74..0000000000 --- a/test/lint/lint-include-guards.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check include guards. - -export LC_ALL=C -HEADER_ID_PREFIX="BITCOIN_" -HEADER_ID_SUFFIX="_H" - -REGEXP_EXCLUDE_FILES_WITH_PREFIX="src/(crypto/ctaes/|leveldb/|crc32c/|secp256k1/|minisketch/|test/fuzz/FuzzedDataProvider.h|tinyformat.h|bench/nanobench.h|univalue/)" - -EXIT_CODE=0 -for HEADER_FILE in $(git ls-files -- "*.h" | grep -vE "^${REGEXP_EXCLUDE_FILES_WITH_PREFIX}") -do - HEADER_ID_BASE=$(cut -f2- -d/ <<< "${HEADER_FILE}" | sed "s/\.h$//g" | tr / _ | tr - _ | tr "[:lower:]" "[:upper:]") - HEADER_ID="${HEADER_ID_PREFIX}${HEADER_ID_BASE}${HEADER_ID_SUFFIX}" - if [[ $(grep --count --extended-regexp "^#(ifndef|define|endif //) ${HEADER_ID}" "${HEADER_FILE}") != 3 ]]; then - echo "${HEADER_FILE} seems to be missing the expected include guard:" - echo " #ifndef ${HEADER_ID}" - echo " #define ${HEADER_ID}" - echo " ..." - echo " #endif // ${HEADER_ID}" - echo - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-includes.py b/test/lint/lint-includes.py new file mode 100755 index 0000000000..ae62994642 --- /dev/null +++ b/test/lint/lint-includes.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for duplicate includes. +# Guard against accidental introduction of new Boost dependencies. +# Check includes: Check for duplicate includes. Enforce bracket syntax includes. + +import os +import re +import sys + +from subprocess import check_output, CalledProcessError + + +EXCLUDED_DIRS = ["src/leveldb/", + "src/crc32c/", + "src/secp256k1/", + "src/minisketch/", + "src/univalue/"] + +EXPECTED_BOOST_INCLUDES = ["boost/algorithm/string/replace.hpp", + "boost/date_time/posix_time/posix_time.hpp", + "boost/multi_index/hashed_index.hpp", + "boost/multi_index/ordered_index.hpp", + "boost/multi_index/sequenced_index.hpp", + "boost/multi_index_container.hpp", + "boost/process.hpp", + "boost/signals2/connection.hpp", + "boost/signals2/optional_last_value.hpp", + "boost/signals2/signal.hpp", + "boost/test/included/unit_test.hpp", + "boost/test/unit_test.hpp"] + + +def get_toplevel(): + return check_output(["git", "rev-parse", "--show-toplevel"], universal_newlines=True, encoding="utf8").rstrip("\n") + + +def list_files_by_suffix(suffixes): + exclude_args = [":(exclude)" + dir for dir in EXCLUDED_DIRS] + + files_list = check_output(["git", "ls-files", "src"] + exclude_args, universal_newlines=True, encoding="utf8").splitlines() + + return [file for file in files_list if file.endswith(suffixes)] + + +def find_duplicate_includes(include_list): + tempset = set() + duplicates = set() + + for inclusion in include_list: + if inclusion in tempset: + duplicates.add(inclusion) + else: + tempset.add(inclusion) + + return duplicates + + +def find_included_cpps(): + included_cpps = list() + + try: + included_cpps = check_output(["git", "grep", "-E", r"^#include [<\"][^>\"]+\.cpp[>\"]", "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + return included_cpps + + +def find_extra_boosts(): + included_boosts = list() + filtered_included_boost_set = set() + exclusion_set = set() + + try: + included_boosts = check_output(["git", "grep", "-E", r"^#include <boost/", "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + for boost in included_boosts: + filtered_included_boost_set.add(re.findall(r'(?<=\<).+?(?=\>)', boost)[0]) + + for expected_boost in EXPECTED_BOOST_INCLUDES: + for boost in filtered_included_boost_set: + if expected_boost in boost: + exclusion_set.add(boost) + + extra_boosts = set(filtered_included_boost_set.difference(exclusion_set)) + + return extra_boosts + + +def find_quote_syntax_inclusions(): + exclude_args = [":(exclude)" + dir for dir in EXCLUDED_DIRS] + quote_syntax_inclusions = list() + + try: + quote_syntax_inclusions = check_output(["git", "grep", r"^#include \"", "--", "*.cpp", "*.h"] + exclude_args, universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + return quote_syntax_inclusions + + +def main(): + exit_code = 0 + + os.chdir(get_toplevel()) + + # Check for duplicate includes + for filename in list_files_by_suffix((".cpp", ".h")): + with open(filename, "r", encoding="utf8") as file: + include_list = [line.rstrip("\n") for line in file if re.match(r"^#include", line)] + + duplicates = find_duplicate_includes(include_list) + + if duplicates: + print(f"Duplicate include(s) in {filename}:") + for duplicate in duplicates: + print(duplicate) + print("") + exit_code = 1 + + # Check if code includes .cpp-files + included_cpps = find_included_cpps() + + if included_cpps: + print("The following files #include .cpp files:") + for included_cpp in included_cpps: + print(included_cpp) + print("") + exit_code = 1 + + # Guard against accidental introduction of new Boost dependencies + extra_boosts = find_extra_boosts() + + if extra_boosts: + for boost in extra_boosts: + print(f"A new Boost dependency in the form of \"{boost}\" appears to have been introduced:") + print(check_output(["git", "grep", boost, "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8")) + exit_code = 1 + + # Check if Boost dependencies are no longer used + for expected_boost in EXPECTED_BOOST_INCLUDES: + try: + check_output(["git", "grep", "-q", r"^#include <%s>" % expected_boost, "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8") + except CalledProcessError as e: + if e.returncode > 1: + raise e + else: + print(f"Good job! The Boost dependency \"{expected_boost}\" is no longer used. " + "Please remove it from EXPECTED_BOOST_INCLUDES in test/lint/lint-includes.py " + "to make sure this dependency is not accidentally reintroduced.\n") + exit_code = 1 + + # Enforce bracket syntax includes + quote_syntax_inclusions = find_quote_syntax_inclusions() + + if quote_syntax_inclusions: + print("Please use bracket syntax includes (\"#include <foo.h>\") instead of quote syntax includes:") + for quote_syntax_inclusion in quote_syntax_inclusions: + print(quote_syntax_inclusion) + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh deleted file mode 100755 index 9e72831ee9..0000000000 --- a/test/lint/lint-includes.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for duplicate includes. -# Guard against accidental introduction of new Boost dependencies. -# Check includes: Check for duplicate includes. Enforce bracket syntax includes. - -export LC_ALL=C -IGNORE_REGEXP="/(leveldb|secp256k1|minisketch|univalue|crc32c)/" - -# cd to root folder of git repo for git ls-files to work properly -cd "$(dirname "$0")/../.." || exit 1 - -filter_suffix() { - git ls-files | grep -E "^src/.*\.${1}"'$' | grep -Ev "${IGNORE_REGEXP}" -} - -EXIT_CODE=0 - -for HEADER_FILE in $(filter_suffix h); do - DUPLICATE_INCLUDES_IN_HEADER_FILE=$(grep -E "^#include " < "${HEADER_FILE}" | sort | uniq -d) - if [[ ${DUPLICATE_INCLUDES_IN_HEADER_FILE} != "" ]]; then - echo "Duplicate include(s) in ${HEADER_FILE}:" - echo "${DUPLICATE_INCLUDES_IN_HEADER_FILE}" - echo - EXIT_CODE=1 - fi -done - -for CPP_FILE in $(filter_suffix cpp); do - DUPLICATE_INCLUDES_IN_CPP_FILE=$(grep -E "^#include " < "${CPP_FILE}" | sort | uniq -d) - if [[ ${DUPLICATE_INCLUDES_IN_CPP_FILE} != "" ]]; then - echo "Duplicate include(s) in ${CPP_FILE}:" - echo "${DUPLICATE_INCLUDES_IN_CPP_FILE}" - echo - EXIT_CODE=1 - fi -done - -INCLUDED_CPP_FILES=$(git grep -E "^#include [<\"][^>\"]+\.cpp[>\"]" -- "*.cpp" "*.h") -if [[ ${INCLUDED_CPP_FILES} != "" ]]; then - echo "The following files #include .cpp files:" - echo "${INCLUDED_CPP_FILES}" - echo - EXIT_CODE=1 -fi - -EXPECTED_BOOST_INCLUDES=( - boost/algorithm/string.hpp - boost/algorithm/string/classification.hpp - boost/algorithm/string/replace.hpp - boost/algorithm/string/split.hpp - boost/date_time/posix_time/posix_time.hpp - boost/multi_index/hashed_index.hpp - boost/multi_index/ordered_index.hpp - boost/multi_index/sequenced_index.hpp - boost/multi_index_container.hpp - boost/process.hpp - boost/signals2/connection.hpp - boost/signals2/optional_last_value.hpp - boost/signals2/signal.hpp - boost/test/included/unit_test.hpp - boost/test/unit_test.hpp -) - -for BOOST_INCLUDE in $(git grep '^#include <boost/' -- "*.cpp" "*.h" | cut -f2 -d: | cut -f2 -d'<' | cut -f1 -d'>' | sort -u); do - IS_EXPECTED_INCLUDE=0 - for EXPECTED_BOOST_INCLUDE in "${EXPECTED_BOOST_INCLUDES[@]}"; do - if [[ "${BOOST_INCLUDE}" == "${EXPECTED_BOOST_INCLUDE}" ]]; then - IS_EXPECTED_INCLUDE=1 - break - fi - done - if [[ ${IS_EXPECTED_INCLUDE} == 0 ]]; then - EXIT_CODE=1 - echo "A new Boost dependency in the form of \"${BOOST_INCLUDE}\" appears to have been introduced:" - git grep "${BOOST_INCLUDE}" -- "*.cpp" "*.h" - echo - fi -done - -for EXPECTED_BOOST_INCLUDE in "${EXPECTED_BOOST_INCLUDES[@]}"; do - if ! git grep -q "^#include <${EXPECTED_BOOST_INCLUDE}>" -- "*.cpp" "*.h"; then - echo "Good job! The Boost dependency \"${EXPECTED_BOOST_INCLUDE}\" is no longer used." - echo "Please remove it from EXPECTED_BOOST_INCLUDES in $0" - echo "to make sure this dependency is not accidentally reintroduced." - echo - EXIT_CODE=1 - fi -done - -QUOTE_SYNTAX_INCLUDES=$(git grep '^#include "' -- "*.cpp" "*.h" | grep -Ev "${IGNORE_REGEXP}") -if [[ ${QUOTE_SYNTAX_INCLUDES} != "" ]]; then - echo "Please use bracket syntax includes (\"#include <foo.h>\") instead of quote syntax includes:" - echo "${QUOTE_SYNTAX_INCLUDES}" - echo - EXIT_CODE=1 -fi - -exit ${EXIT_CODE} diff --git a/test/lint/lint-locale-dependence.py b/test/lint/lint-locale-dependence.py new file mode 100755 index 0000000000..9b2cf4587a --- /dev/null +++ b/test/lint/lint-locale-dependence.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Be aware that bitcoind and bitcoin-qt differ in terms of localization: Qt +# opts in to POSIX localization by running setlocale(LC_ALL, "") on startup, +# whereas no such call is made in bitcoind. +# +# Qt runs setlocale(LC_ALL, "") on initialization. This installs the locale +# specified by the user's LC_ALL (or LC_*) environment variable as the new +# C locale. +# +# In contrast, bitcoind does not opt in to localization -- no call to +# setlocale(LC_ALL, "") is made and the environment variables LC_* are +# thus ignored. +# +# This results in situations where bitcoind is guaranteed to be running +# with the classic locale ("C") whereas the locale of bitcoin-qt will vary +# depending on the user's environment variables. +# +# An example: Assuming the environment variable LC_ALL=de_DE then the +# call std::to_string(1.23) will return "1.230000" in bitcoind but +# "1,230000" in bitcoin-qt. +# +# From the Qt documentation: +# "On Unix/Linux Qt is configured to use the system locale settings by default. +# This can cause a conflict when using POSIX functions, for instance, when +# converting between data types such as floats and strings, since the notation +# may differ between locales. To get around this problem, call the POSIX function +# setlocale(LC_NUMERIC,"C") right after initializing QApplication, QGuiApplication +# or QCoreApplication to reset the locale that is used for number formatting to +# "C"-locale." +# +# See https://doc.qt.io/qt-5/qcoreapplication.html#locale-settings and +# https://stackoverflow.com/a/34878283 for more details. +# +# TODO: Reduce KNOWN_VIOLATIONS by replacing uses of locale dependent snprintf with strprintf. + +import re +import sys + +from subprocess import check_output, CalledProcessError + + +KNOWN_VIOLATIONS = [ + "src/dbwrapper.cpp:.*vsnprintf", + "src/test/dbwrapper_tests.cpp:.*snprintf", + "src/test/fuzz/locale.cpp:.*setlocale", + "src/test/fuzz/string.cpp:.*strtol", + "src/test/fuzz/string.cpp:.*strtoul", + "src/test/util_tests.cpp:.*strtoll", + "src/wallet/bdb.cpp:.*DbEnv::strerror", # False positive + "src/util/syserror.cpp:.*strerror", # Outside this function use `SysErrorString` +] + +REGEXP_EXTERNAL_DEPENDENCIES_EXCLUSIONS = [ + "src/crypto/ctaes/", + "src/leveldb/", + "src/secp256k1/", + "src/minisketch/", + "src/tinyformat.h", + "src/univalue/" +] + +LOCALE_DEPENDENT_FUNCTIONS = [ + "alphasort", # LC_COLLATE (via strcoll) + "asctime", # LC_TIME (directly) + "asprintf", # (via vasprintf) + "atof", # LC_NUMERIC (via strtod) + "atoi", # LC_NUMERIC (via strtol) + "atol", # LC_NUMERIC (via strtol) + "atoll", # (via strtoll) + "atoq", + "btowc", # LC_CTYPE (directly) + "ctime", # (via asctime or localtime) + "dprintf", # (via vdprintf) + "fgetwc", + "fgetws", + "fold_case", # boost::locale::fold_case + "fprintf", # (via vfprintf) + "fputwc", + "fputws", + "fscanf", # (via __vfscanf) + "fwprintf", # (via __vfwprintf) + "getdate", # via __getdate_r => isspace // __localtime_r + "getwc", + "getwchar", + "is_digit", # boost::algorithm::is_digit + "is_space", # boost::algorithm::is_space + "isalnum", # LC_CTYPE + "isalpha", # LC_CTYPE + "isblank", # LC_CTYPE + "iscntrl", # LC_CTYPE + "isctype", # LC_CTYPE + "isdigit", # LC_CTYPE + "isgraph", # LC_CTYPE + "islower", # LC_CTYPE + "isprint", # LC_CTYPE + "ispunct", # LC_CTYPE + "isspace", # LC_CTYPE + "isupper", # LC_CTYPE + "iswalnum", # LC_CTYPE + "iswalpha", # LC_CTYPE + "iswblank", # LC_CTYPE + "iswcntrl", # LC_CTYPE + "iswctype", # LC_CTYPE + "iswdigit", # LC_CTYPE + "iswgraph", # LC_CTYPE + "iswlower", # LC_CTYPE + "iswprint", # LC_CTYPE + "iswpunct", # LC_CTYPE + "iswspace", # LC_CTYPE + "iswupper", # LC_CTYPE + "iswxdigit", # LC_CTYPE + "isxdigit", # LC_CTYPE + "localeconv", # LC_NUMERIC + LC_MONETARY + "mblen", # LC_CTYPE + "mbrlen", + "mbrtowc", + "mbsinit", + "mbsnrtowcs", + "mbsrtowcs", + "mbstowcs", # LC_CTYPE + "mbtowc", # LC_CTYPE + "mktime", + "normalize", # boost::locale::normalize + "printf", # LC_NUMERIC + "putwc", + "putwchar", + "scanf", # LC_NUMERIC + "setlocale", + "snprintf", + "sprintf", + "sscanf", + "std::locale::global", + "std::to_string", + "stod", + "stof", + "stoi", + "stol", + "stold", + "stoll", + "stoul", + "stoull", + "strcasecmp", + "strcasestr", + "strcoll", # LC_COLLATE + "strerror", + "strfmon", + "strftime", # LC_TIME + "strncasecmp", + "strptime", + "strtod", # LC_NUMERIC + "strtof", + "strtoimax", + "strtol", # LC_NUMERIC + "strtold", + "strtoll", + "strtoq", + "strtoul", # LC_NUMERIC + "strtoull", + "strtoumax", + "strtouq", + "strxfrm", # LC_COLLATE + "swprintf", + "to_lower", # boost::locale::to_lower + "to_title", # boost::locale::to_title + "to_upper", # boost::locale::to_upper + "tolower", # LC_CTYPE + "toupper", # LC_CTYPE + "towctrans", + "towlower", # LC_CTYPE + "towupper", # LC_CTYPE + "trim", # boost::algorithm::trim + "trim_left", # boost::algorithm::trim_left + "trim_right", # boost::algorithm::trim_right + "ungetwc", + "vasprintf", + "vdprintf", + "versionsort", + "vfprintf", + "vfscanf", + "vfwprintf", + "vprintf", + "vscanf", + "vsnprintf", + "vsprintf", + "vsscanf", + "vswprintf", + "vwprintf", + "wcrtomb", + "wcscasecmp", + "wcscoll", # LC_COLLATE + "wcsftime", # LC_TIME + "wcsncasecmp", + "wcsnrtombs", + "wcsrtombs", + "wcstod", # LC_NUMERIC + "wcstof", + "wcstoimax", + "wcstol", # LC_NUMERIC + "wcstold", + "wcstoll", + "wcstombs", # LC_CTYPE + "wcstoul", # LC_NUMERIC + "wcstoull", + "wcstoumax", + "wcswidth", + "wcsxfrm", # LC_COLLATE + "wctob", + "wctomb", # LC_CTYPE + "wctrans", + "wctype", + "wcwidth", + "wprintf" +] + + +def find_locale_dependent_function_uses(): + regexp_locale_dependent_functions = "|".join(LOCALE_DEPENDENT_FUNCTIONS) + exclude_args = [":(exclude)" + excl for excl in REGEXP_EXTERNAL_DEPENDENCIES_EXCLUSIONS] + git_grep_command = ["git", "grep", "-E", "[^a-zA-Z0-9_\\`'\"<>](" + regexp_locale_dependent_functions + ")(_r|_s)?[^a-zA-Z0-9_\\`'\"<>]", "--", "*.cpp", "*.h"] + exclude_args + git_grep_output = list() + + try: + git_grep_output = check_output(git_grep_command, universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + return git_grep_output + + +def main(): + exit_code = 0 + + regexp_ignore_known_violations = "|".join(KNOWN_VIOLATIONS) + git_grep_output = find_locale_dependent_function_uses() + + for locale_dependent_function in LOCALE_DEPENDENT_FUNCTIONS: + matches = [line for line in git_grep_output + if re.search("[^a-zA-Z0-9_\\`'\"<>]" + locale_dependent_function + "(_r|_s)?[^a-zA-Z0-9_\\`'\"<>]", line) + and not re.search("\\.(c|cpp|h):\\s*(//|\\*|/\\*|\").*" + locale_dependent_function, line) + and not re.search(regexp_ignore_known_violations, line)] + if matches: + print(f"The locale dependent function {locale_dependent_function}(...) appears to be used:") + for match in matches: + print(match) + print("") + exit_code = 1 + + if exit_code == 1: + print("Unnecessary locale depedence can cause bugs that are very tricky to isolate and fix. Please avoid using locale dependent functions if possible.\n") + print(f"Advice not applicable in this specific case? Add an exception by updating the ignore list in {sys.argv[0]}") + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh deleted file mode 100755 index 7d608eed6a..0000000000 --- a/test/lint/lint-locale-dependence.sh +++ /dev/null @@ -1,241 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C - -# Be aware that bitcoind and bitcoin-qt differ in terms of localization: Qt -# opts in to POSIX localization by running setlocale(LC_ALL, "") on startup, -# whereas no such call is made in bitcoind. -# -# Qt runs setlocale(LC_ALL, "") on initialization. This installs the locale -# specified by the user's LC_ALL (or LC_*) environment variable as the new -# C locale. -# -# In contrast, bitcoind does not opt in to localization -- no call to -# setlocale(LC_ALL, "") is made and the environment variables LC_* are -# thus ignored. -# -# This results in situations where bitcoind is guaranteed to be running -# with the classic locale ("C") whereas the locale of bitcoin-qt will vary -# depending on the user's environment variables. -# -# An example: Assuming the environment variable LC_ALL=de_DE then the -# call std::to_string(1.23) will return "1.230000" in bitcoind but -# "1,230000" in bitcoin-qt. -# -# From the Qt documentation: -# "On Unix/Linux Qt is configured to use the system locale settings by default. -# This can cause a conflict when using POSIX functions, for instance, when -# converting between data types such as floats and strings, since the notation -# may differ between locales. To get around this problem, call the POSIX function -# setlocale(LC_NUMERIC,"C") right after initializing QApplication, QGuiApplication -# or QCoreApplication to reset the locale that is used for number formatting to -# "C"-locale." -# -# See https://doc.qt.io/qt-5/qcoreapplication.html#locale-settings and -# https://stackoverflow.com/a/34878283 for more details. - -# TODO: Reduce KNOWN_VIOLATIONS by replacing uses of locale dependent snprintf with strprintf. -KNOWN_VIOLATIONS=( - "src/dbwrapper.cpp:.*vsnprintf" - "src/test/dbwrapper_tests.cpp:.*snprintf" - "src/test/fuzz/locale.cpp" - "src/test/fuzz/string.cpp" - "src/test/util_tests.cpp" -) - -REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(crypto/ctaes/|leveldb/|secp256k1/|minisketch/|tinyformat.h|univalue/)" - -LOCALE_DEPENDENT_FUNCTIONS=( - alphasort # LC_COLLATE (via strcoll) - asctime # LC_TIME (directly) - asprintf # (via vasprintf) - atof # LC_NUMERIC (via strtod) - atoi # LC_NUMERIC (via strtol) - atol # LC_NUMERIC (via strtol) - atoll # (via strtoll) - atoq - btowc # LC_CTYPE (directly) - ctime # (via asctime or localtime) - dprintf # (via vdprintf) - fgetwc - fgetws - fold_case # boost::locale::fold_case - fprintf # (via vfprintf) - fputwc - fputws - fscanf # (via __vfscanf) - fwprintf # (via __vfwprintf) - getdate # via __getdate_r => isspace // __localtime_r - getwc - getwchar - is_digit # boost::algorithm::is_digit - is_space # boost::algorithm::is_space - isalnum # LC_CTYPE - isalpha # LC_CTYPE - isblank # LC_CTYPE - iscntrl # LC_CTYPE - isctype # LC_CTYPE - isdigit # LC_CTYPE - isgraph # LC_CTYPE - islower # LC_CTYPE - isprint # LC_CTYPE - ispunct # LC_CTYPE - isspace # LC_CTYPE - isupper # LC_CTYPE - iswalnum # LC_CTYPE - iswalpha # LC_CTYPE - iswblank # LC_CTYPE - iswcntrl # LC_CTYPE - iswctype # LC_CTYPE - iswdigit # LC_CTYPE - iswgraph # LC_CTYPE - iswlower # LC_CTYPE - iswprint # LC_CTYPE - iswpunct # LC_CTYPE - iswspace # LC_CTYPE - iswupper # LC_CTYPE - iswxdigit # LC_CTYPE - isxdigit # LC_CTYPE - localeconv # LC_NUMERIC + LC_MONETARY - mblen # LC_CTYPE - mbrlen - mbrtowc - mbsinit - mbsnrtowcs - mbsrtowcs - mbstowcs # LC_CTYPE - mbtowc # LC_CTYPE - mktime - normalize # boost::locale::normalize - printf # LC_NUMERIC - putwc - putwchar - scanf # LC_NUMERIC - setlocale - snprintf - sprintf - sscanf - std::locale::global - std::to_string - stod - stof - stoi - stol - stold - stoll - stoul - stoull - strcasecmp - strcasestr - strcoll # LC_COLLATE -# strerror - strfmon - strftime # LC_TIME - strncasecmp - strptime - strtod # LC_NUMERIC - strtof - strtoimax - strtol # LC_NUMERIC - strtold - strtoll - strtoq - strtoul # LC_NUMERIC - strtoull - strtoumax - strtouq - strxfrm # LC_COLLATE - swprintf - to_lower # boost::locale::to_lower - to_title # boost::locale::to_title - to_upper # boost::locale::to_upper - tolower # LC_CTYPE - toupper # LC_CTYPE - towctrans - towlower # LC_CTYPE - towupper # LC_CTYPE - trim # boost::algorithm::trim - trim_left # boost::algorithm::trim_left - trim_right # boost::algorithm::trim_right - ungetwc - vasprintf - vdprintf - versionsort - vfprintf - vfscanf - vfwprintf - vprintf - vscanf - vsnprintf - vsprintf - vsscanf - vswprintf - vwprintf - wcrtomb - wcscasecmp - wcscoll # LC_COLLATE - wcsftime # LC_TIME - wcsncasecmp - wcsnrtombs - wcsrtombs - wcstod # LC_NUMERIC - wcstof - wcstoimax - wcstol # LC_NUMERIC - wcstold - wcstoll - wcstombs # LC_CTYPE - wcstoul # LC_NUMERIC - wcstoull - wcstoumax - wcswidth - wcsxfrm # LC_COLLATE - wctob - wctomb # LC_CTYPE - wctrans - wctype - wcwidth - wprintf -) - -function join_array { - local IFS="$1" - shift - echo "$*" -} - -REGEXP_IGNORE_KNOWN_VIOLATIONS=$(join_array "|" "${KNOWN_VIOLATIONS[@]}") - -# Invoke "git grep" only once in order to minimize run-time -REGEXP_LOCALE_DEPENDENT_FUNCTIONS=$(join_array "|" "${LOCALE_DEPENDENT_FUNCTIONS[@]}") -GIT_GREP_OUTPUT=$(git grep -E "[^a-zA-Z0-9_\`'\"<>](${REGEXP_LOCALE_DEPENDENT_FUNCTIONS}(_r|_s)?)[^a-zA-Z0-9_\`'\"<>]" -- "*.cpp" "*.h") - -EXIT_CODE=0 -for LOCALE_DEPENDENT_FUNCTION in "${LOCALE_DEPENDENT_FUNCTIONS[@]}"; do - MATCHES=$(grep -E "[^a-zA-Z0-9_\`'\"<>]${LOCALE_DEPENDENT_FUNCTION}(_r|_s)?[^a-zA-Z0-9_\`'\"<>]" <<< "${GIT_GREP_OUTPUT}" | \ - grep -vE "\.(c|cpp|h):\s*(//|\*|/\*|\").*${LOCALE_DEPENDENT_FUNCTION}") - if [[ ${REGEXP_IGNORE_EXTERNAL_DEPENDENCIES} != "" ]]; then - MATCHES=$(grep -vE "${REGEXP_IGNORE_EXTERNAL_DEPENDENCIES}" <<< "${MATCHES}") - fi - if [[ ${REGEXP_IGNORE_KNOWN_VIOLATIONS} != "" ]]; then - MATCHES=$(grep -vE "${REGEXP_IGNORE_KNOWN_VIOLATIONS}" <<< "${MATCHES}") - fi - if [[ ${MATCHES} != "" ]]; then - echo "The locale dependent function ${LOCALE_DEPENDENT_FUNCTION}(...) appears to be used:" - echo "${MATCHES}" - echo - EXIT_CODE=1 - fi -done -if [[ ${EXIT_CODE} != 0 ]]; then - echo "Unnecessary locale dependence can cause bugs that are very" - echo "tricky to isolate and fix. Please avoid using locale dependent" - echo "functions if possible." - echo - echo "Advice not applicable in this specific case? Add an exception" - echo "by updating the ignore list in $0" -fi -exit ${EXIT_CODE} diff --git a/test/lint/lint-logs.py b/test/lint/lint-logs.py new file mode 100755 index 0000000000..e6c4c068fb --- /dev/null +++ b/test/lint/lint-logs.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check that all logs are terminated with '\n' +# +# Some logs are continued over multiple lines. They should be explicitly +# commented with /* Continued */ + +import re +import sys + +from subprocess import check_output + + +def main(): + logs_list = check_output(["git", "grep", "--extended-regexp", r"LogPrintf?\(", "--", "*.cpp"], universal_newlines=True, encoding="utf8").splitlines() + + unterminated_logs = [line for line in logs_list if not re.search(r'(\\n"|/\* Continued \*/)', line)] + + if unterminated_logs != []: + print("All calls to LogPrintf() and LogPrint() should be terminated with \\n") + print("") + + for line in unterminated_logs: + print(line) + + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-logs.sh b/test/lint/lint-logs.sh deleted file mode 100755 index 6d5165f649..0000000000 --- a/test/lint/lint-logs.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check that all logs are terminated with '\n' -# -# Some logs are continued over multiple lines. They should be explicitly -# commented with /* Continued */ -# -# There are some instances of LogPrintf() in comments. Those can be -# ignored - -export LC_ALL=C -UNTERMINATED_LOGS=$(git grep --extended-regexp "LogPrintf?\(" -- "*.cpp" | \ - grep -v '\\n"' | \ - grep -v '\.\.\.' | \ - grep -v "/\* Continued \*/" | \ - grep -v "LogPrint()" | \ - grep -v "LogPrintf()") -if [[ ${UNTERMINATED_LOGS} != "" ]]; then - # shellcheck disable=SC2028 - echo "All calls to LogPrintf() and LogPrint() should be terminated with \\n" - echo - echo "${UNTERMINATED_LOGS}" - exit 1 -fi diff --git a/test/lint/lint-python-dead-code.py b/test/lint/lint-python-dead-code.py new file mode 100755 index 0000000000..b3f9394788 --- /dev/null +++ b/test/lint/lint-python-dead-code.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Find dead Python code. +""" + +from subprocess import check_output, STDOUT, CalledProcessError + +FILES_ARGS = ['git', 'ls-files', '--', '*.py'] + + +def check_vulture_install(): + try: + check_output(["vulture", "--version"]) + except FileNotFoundError: + print("Skipping Python dead code linting since vulture is not installed. Install by running \"pip3 install vulture\"") + exit(0) + + +def main(): + check_vulture_install() + + files = check_output(FILES_ARGS).decode("utf-8").splitlines() + # --min-confidence 100 will only report code that is guaranteed to be unused within the analyzed files. + # Any value below 100 introduces the risk of false positives, which would create an unacceptable maintenance burden. + vulture_args = ['vulture', '--min-confidence=100'] + files + + try: + check_output(vulture_args, stderr=STDOUT) + except CalledProcessError as e: + print(e.output.decode("utf-8"), end="") + print("Python dead code detection found some issues") + exit(1) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-python-dead-code.sh b/test/lint/lint-python-dead-code.sh deleted file mode 100755 index 247bfb310a..0000000000 --- a/test/lint/lint-python-dead-code.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Find dead Python code. - -export LC_ALL=C - -if ! command -v vulture > /dev/null; then - echo "Skipping Python dead code linting since vulture is not installed. Install by running \"pip3 install vulture\"" - exit 0 -fi - -# --min-confidence 100 will only report code that is guaranteed to be unused within the analyzed files. -# Any value below 100 introduces the risk of false positives, which would create an unacceptable maintenance burden. -mapfile -t FILES < <(git ls-files -- "*.py") -if ! vulture --min-confidence 100 "${FILES[@]}"; then - echo "Python dead code detection found some issues" - exit 1 -fi diff --git a/test/lint/lint-python-mutable-default-parameters.py b/test/lint/lint-python-mutable-default-parameters.py new file mode 100755 index 0000000000..7991e3630b --- /dev/null +++ b/test/lint/lint-python-mutable-default-parameters.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Detect when a mutable list or dict is used as a default parameter value in a Python function. +""" + +import subprocess +import sys + + +def main(): + command = [ + "git", + "grep", + "-E", + r"^\s*def [a-zA-Z0-9_]+\(.*=\s*(\[|\{)", + "--", + "*.py", + ] + output = subprocess.run(command, stdout=subprocess.PIPE, universal_newlines=True) + if len(output.stdout) > 0: + error_msg = ( + "A mutable list or dict seems to be used as default parameter value:\n\n" + f"{output.stdout}\n" + f"{example()}" + ) + print(error_msg) + sys.exit(1) + else: + sys.exit(0) + + +def example(): + return """This is how mutable list and dict default parameter values behave: + +>>> def f(i, j=[], k={}): +... j.append(i) +... k[i] = True +... return j, k +... +>>> f(1) +([1], {1: True}) +>>> f(1) +([1, 1], {1: True}) +>>> f(2) +([1, 1, 2], {1: True, 2: True}) + +The intended behaviour was likely: + +>>> def f(i, j=None, k=None): +... if j is None: +... j = [] +... if k is None: +... k = {} +... j.append(i) +... k[i] = True +... return j, k +... +>>> f(1) +([1], {1: True}) +>>> f(1) +([1], {1: True}) +>>> f(2) +([2], {2: True})""" + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-python-mutable-default-parameters.sh b/test/lint/lint-python-mutable-default-parameters.sh deleted file mode 100755 index 1f9f035d30..0000000000 --- a/test/lint/lint-python-mutable-default-parameters.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Detect when a mutable list or dict is used as a default parameter value in a Python function. - -export LC_ALL=C -EXIT_CODE=0 -OUTPUT=$(git grep -E '^\s*def [a-zA-Z0-9_]+\(.*=\s*(\[|\{)' -- "*.py") -if [[ ${OUTPUT} != "" ]]; then - echo "A mutable list or dict seems to be used as default parameter value:" - echo - echo "${OUTPUT}" - echo - cat << EXAMPLE -This is how mutable list and dict default parameter values behave: - ->>> def f(i, j=[], k={}): -... j.append(i) -... k[i] = True -... return j, k -... ->>> f(1) -([1], {1: True}) ->>> f(1) -([1, 1], {1: True}) ->>> f(2) -([1, 1, 2], {1: True, 2: True}) - -The intended behaviour was likely: - ->>> def f(i, j=None, k=None): -... if j is None: -... j = [] -... if k is None: -... k = {} -... j.append(i) -... k[i] = True -... return j, k -... ->>> f(1) -([1], {1: True}) ->>> f(1) -([1], {1: True}) ->>> f(2) -([2], {2: True}) -EXAMPLE - EXIT_CODE=1 -fi -exit ${EXIT_CODE} diff --git a/test/lint/lint-python-utf8-encoding.py b/test/lint/lint-python-utf8-encoding.py new file mode 100755 index 0000000000..62fdc34d50 --- /dev/null +++ b/test/lint/lint-python-utf8-encoding.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Make sure we explicitly open all text files using UTF-8 (or ASCII) encoding to +# avoid potential issues on the BSDs where the locale is not always set. + +import sys +import re + +from subprocess import check_output, CalledProcessError + +EXCLUDED_DIRS = ["src/crc32c/"] + + +def get_exclude_args(): + return [":(exclude)" + dir for dir in EXCLUDED_DIRS] + + +def check_fileopens(): + fileopens = list() + + try: + fileopens = check_output(["git", "grep", r" open(", "--", "*.py"] + get_exclude_args(), universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + filtered_fileopens = [fileopen for fileopen in fileopens if not re.search(r"encoding=.(ascii|utf8|utf-8).|open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]", fileopen)] + + return filtered_fileopens + + +def check_checked_outputs(): + checked_outputs = list() + + try: + checked_outputs = check_output(["git", "grep", "check_output(", "--", "*.py"] + get_exclude_args(), universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + filtered_checked_outputs = [checked_output for checked_output in checked_outputs if re.search(r"universal_newlines=True", checked_output) and not re.search(r"encoding=.(ascii|utf8|utf-8).", checked_output)] + + return filtered_checked_outputs + + +def main(): + exit_code = 0 + + nonexplicit_utf8_fileopens = check_fileopens() + if nonexplicit_utf8_fileopens: + print("Python's open(...) seems to be used to open text files without explicitly specifying encoding='utf8':\n") + for fileopen in nonexplicit_utf8_fileopens: + print(fileopen) + exit_code = 1 + + nonexplicit_utf8_checked_outputs = check_checked_outputs() + if nonexplicit_utf8_checked_outputs: + if nonexplicit_utf8_fileopens: + print("\n") + print("Python's check_output(...) seems to be used to get program outputs without explicitly specifying encoding='utf8':\n") + for checked_output in nonexplicit_utf8_checked_outputs: + print(checked_output) + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-python-utf8-encoding.sh b/test/lint/lint-python-utf8-encoding.sh deleted file mode 100755 index 6e5b18fc23..0000000000 --- a/test/lint/lint-python-utf8-encoding.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Make sure we explicitly open all text files using UTF-8 (or ASCII) encoding to -# avoid potential issues on the BSDs where the locale is not always set. - -export LC_ALL=C -EXIT_CODE=0 -OUTPUT=$(git grep " open(" -- "*.py" ":(exclude)src/crc32c/" | grep -vE "encoding=.(ascii|utf8|utf-8)." | grep -vE "open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]") -if [[ ${OUTPUT} != "" ]]; then - echo "Python's open(...) seems to be used to open text files without explicitly" - echo "specifying encoding=\"utf8\":" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi -OUTPUT=$(git grep "check_output(" -- "*.py" ":(exclude)src/crc32c/"| grep "universal_newlines=True" | grep -vE "encoding=.(ascii|utf8|utf-8).") -if [[ ${OUTPUT} != "" ]]; then - echo "Python's check_output(...) seems to be used to get program outputs without explicitly" - echo "specifying encoding=\"utf8\":" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi -exit ${EXIT_CODE} diff --git a/test/lint/lint-python.py b/test/lint/lint-python.py new file mode 100755 index 0000000000..4d16facfea --- /dev/null +++ b/test/lint/lint-python.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check for specified flake8 and mypy warnings in python files. +""" + +import os +import pkg_resources +import subprocess +import sys + +DEPS = ['flake8', 'mypy', 'pyzmq'] +MYPY_CACHE_DIR = f"{os.getenv('BASE_ROOT_DIR', '')}/test/.mypy_cache" +FILES_ARGS = ['git', 'ls-files', 'test/functional/*.py', 'contrib/devtools/*.py'] + +ENABLED = ( + 'E101,' # indentation contains mixed spaces and tabs + 'E112,' # expected an indented block + 'E113,' # unexpected indentation + 'E115,' # expected an indented block (comment) + 'E116,' # unexpected indentation (comment) + 'E125,' # continuation line with same indent as next logical line + 'E129,' # visually indented line with same indent as next logical line + 'E131,' # continuation line unaligned for hanging indent + 'E133,' # closing bracket is missing indentation + 'E223,' # tab before operator + 'E224,' # tab after operator + 'E242,' # tab after ',' + 'E266,' # too many leading '#' for block comment + 'E271,' # multiple spaces after keyword + 'E272,' # multiple spaces before keyword + 'E273,' # tab after keyword + 'E274,' # tab before keyword + 'E275,' # missing whitespace after keyword + 'E304,' # blank lines found after function decorator + 'E306,' # expected 1 blank line before a nested definition + 'E401,' # multiple imports on one line + 'E402,' # module level import not at top of file + 'E502,' # the backslash is redundant between brackets + 'E701,' # multiple statements on one line (colon) + 'E702,' # multiple statements on one line (semicolon) + 'E703,' # statement ends with a semicolon + 'E711,' # comparison to None should be 'if cond is None:' + 'E714,' # test for object identity should be "is not" + 'E721,' # do not compare types, use "isinstance()" + 'E742,' # do not define classes named "l", "O", or "I" + 'E743,' # do not define functions named "l", "O", or "I" + 'E901,' # SyntaxError: invalid syntax + 'E902,' # TokenError: EOF in multi-line string + 'F401,' # module imported but unused + 'F402,' # import module from line N shadowed by loop variable + 'F403,' # 'from foo_module import *' used; unable to detect undefined names + 'F404,' # future import(s) name after other statements + 'F405,' # foo_function may be undefined, or defined from star imports: bar_module + 'F406,' # "from module import *" only allowed at module level + 'F407,' # an undefined __future__ feature name was imported + 'F601,' # dictionary key name repeated with different values + 'F602,' # dictionary key variable name repeated with different values + 'F621,' # too many expressions in an assignment with star-unpacking + 'F622,' # two or more starred expressions in an assignment (a, *b, *c = d) + 'F631,' # assertion test is a tuple, which are always True + 'F632,' # use ==/!= to compare str, bytes, and int literals + 'F701,' # a break statement outside of a while or for loop + 'F702,' # a continue statement outside of a while or for loop + 'F703,' # a continue statement in a finally block in a loop + 'F704,' # a yield or yield from statement outside of a function + 'F705,' # a return statement with arguments inside a generator + 'F706,' # a return statement outside of a function/method + 'F707,' # an except: block as not the last exception handler + 'F811,' # redefinition of unused name from line N + 'F812,' # list comprehension redefines 'foo' from line N + 'F821,' # undefined name 'Foo' + 'F822,' # undefined name name in __all__ + 'F823,' # local variable name … referenced before assignment + 'F831,' # duplicate argument name in function definition + 'F841,' # local variable 'foo' is assigned to but never used + 'W191,' # indentation contains tabs + 'W291,' # trailing whitespace + 'W292,' # no newline at end of file + 'W293,' # blank line contains whitespace + 'W601,' # .has_key() is deprecated, use "in" + 'W602,' # deprecated form of raising exception + 'W603,' # "<>" is deprecated, use "!=" + 'W604,' # backticks are deprecated, use "repr()" + 'W605,' # invalid escape sequence "x" + 'W606,' # 'async' and 'await' are reserved keywords starting with Python 3.7 +) + + +def check_dependencies(): + working_set = {pkg.key for pkg in pkg_resources.working_set} + + for dep in DEPS: + if dep not in working_set: + print(f"Skipping Python linting since {dep} is not installed.") + exit(0) + + +def main(): + check_dependencies() + + if len(sys.argv) > 1: + flake8_files = sys.argv[1:] + else: + files_args = ['git', 'ls-files', '*.py'] + flake8_files = subprocess.check_output(files_args).decode("utf-8").splitlines() + + flake8_args = ['flake8', '--ignore=B,C,E,F,I,N,W', f'--select={ENABLED}'] + flake8_files + flake8_env = os.environ.copy() + flake8_env["PYTHONWARNINGS"] = "ignore" + + try: + subprocess.check_call(flake8_args, env=flake8_env) + except subprocess.CalledProcessError: + exit(1) + + mypy_files = subprocess.check_output(FILES_ARGS).decode("utf-8").splitlines() + mypy_args = ['mypy', '--show-error-codes'] + mypy_files + + try: + subprocess.check_call(mypy_args) + except subprocess.CalledProcessError: + exit(1) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh deleted file mode 100755 index 7d7857d325..0000000000 --- a/test/lint/lint-python.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2017-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for specified flake8 warnings in python files. - -export LC_ALL=C -export MYPY_CACHE_DIR="${BASE_ROOT_DIR}/test/.mypy_cache" - -enabled=( - E101 # indentation contains mixed spaces and tabs - E112 # expected an indented block - E113 # unexpected indentation - E115 # expected an indented block (comment) - E116 # unexpected indentation (comment) - E125 # continuation line with same indent as next logical line - E129 # visually indented line with same indent as next logical line - E131 # continuation line unaligned for hanging indent - E133 # closing bracket is missing indentation - E223 # tab before operator - E224 # tab after operator - E242 # tab after ',' - E266 # too many leading '#' for block comment - E271 # multiple spaces after keyword - E272 # multiple spaces before keyword - E273 # tab after keyword - E274 # tab before keyword - E275 # missing whitespace after keyword - E304 # blank lines found after function decorator - E306 # expected 1 blank line before a nested definition - E401 # multiple imports on one line - E402 # module level import not at top of file - E502 # the backslash is redundant between brackets - E701 # multiple statements on one line (colon) - E702 # multiple statements on one line (semicolon) - E703 # statement ends with a semicolon - E711 # comparison to None should be 'if cond is None:' - E714 # test for object identity should be "is not" - E721 # do not compare types, use "isinstance()" - E742 # do not define classes named "l", "O", or "I" - E743 # do not define functions named "l", "O", or "I" - E901 # SyntaxError: invalid syntax - E902 # TokenError: EOF in multi-line string - F401 # module imported but unused - F402 # import module from line N shadowed by loop variable - F403 # 'from foo_module import *' used; unable to detect undefined names - F404 # future import(s) name after other statements - F405 # foo_function may be undefined, or defined from star imports: bar_module - F406 # "from module import *" only allowed at module level - F407 # an undefined __future__ feature name was imported - F601 # dictionary key name repeated with different values - F602 # dictionary key variable name repeated with different values - F621 # too many expressions in an assignment with star-unpacking - F622 # two or more starred expressions in an assignment (a, *b, *c = d) - F631 # assertion test is a tuple, which are always True - F632 # use ==/!= to compare str, bytes, and int literals - F701 # a break statement outside of a while or for loop - F702 # a continue statement outside of a while or for loop - F703 # a continue statement in a finally block in a loop - F704 # a yield or yield from statement outside of a function - F705 # a return statement with arguments inside a generator - F706 # a return statement outside of a function/method - F707 # an except: block as not the last exception handler - F811 # redefinition of unused name from line N - F812 # list comprehension redefines 'foo' from line N - F821 # undefined name 'Foo' - F822 # undefined name name in __all__ - F823 # local variable name … referenced before assignment - F831 # duplicate argument name in function definition - F841 # local variable 'foo' is assigned to but never used - W191 # indentation contains tabs - W291 # trailing whitespace - W292 # no newline at end of file - W293 # blank line contains whitespace - W601 # .has_key() is deprecated, use "in" - W602 # deprecated form of raising exception - W603 # "<>" is deprecated, use "!=" - W604 # backticks are deprecated, use "repr()" - W605 # invalid escape sequence "x" - W606 # 'async' and 'await' are reserved keywords starting with Python 3.7 -) - -if ! command -v flake8 > /dev/null; then - echo "Skipping Python linting since flake8 is not installed." - exit 0 -elif PYTHONWARNINGS="ignore" flake8 --version | grep -q "Python 2"; then - echo "Skipping Python linting since flake8 is running under Python 2. Install the Python 3 version of flake8." - exit 0 -fi - -EXIT_CODE=0 - -# shellcheck disable=SC2046 -if ! PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}") $( - if [[ $# == 0 ]]; then - git ls-files "*.py" - else - echo "$@" - fi -); then - EXIT_CODE=1 -fi - -mapfile -t FILES < <(git ls-files "test/functional/*.py" "contrib/devtools/*.py") -if ! mypy --show-error-codes "${FILES[@]}"; then - EXIT_CODE=1 -fi - -exit $EXIT_CODE diff --git a/test/lint/lint-qt.sh b/test/lint/lint-qt.sh deleted file mode 100755 index 2e77682aa2..0000000000 --- a/test/lint/lint-qt.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for SIGNAL/SLOT connect style, removed since Qt4 support drop. - -export LC_ALL=C - -EXIT_CODE=0 - -OUTPUT=$(git grep -E '(SIGNAL|, ?SLOT)\(' -- src/qt) -if [[ ${OUTPUT} != "" ]]; then - echo "Use Qt5 connect style in:" - echo "$OUTPUT" - EXIT_CODE=1 -fi - -exit ${EXIT_CODE} diff --git a/test/lint/lint-shell-locale.py b/test/lint/lint-shell-locale.py new file mode 100755 index 0000000000..f3dfe18a95 --- /dev/null +++ b/test/lint/lint-shell-locale.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Make sure all shell scripts are: +a.) explicitly opt out of locale dependence using + "export LC_ALL=C" or "export LC_ALL=C.UTF-8", or +b.) explicitly opt in to locale dependence using the annotation below. +""" + +import subprocess +import sys +import re + +OPT_IN_LINE = '# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"' + +OPT_OUT_LINES = [ + 'export LC_ALL=C', + 'export LC_ALL=C.UTF-8', +] + +def get_shell_files_list(): + command = [ + 'git', + 'ls-files', + '--', + '*.sh', + ] + try: + return subprocess.check_output(command, stderr = subprocess.STDOUT).decode('utf-8').splitlines() + except subprocess.CalledProcessError as e: + if e.returncode > 1: # return code is 1 when match is empty + print(e.output.decode('utf-8'), end='') + sys.exit(1) + return [] + +def main(): + exit_code = 0 + shell_files = get_shell_files_list() + for file_path in shell_files: + if re.search('src/(secp256k1|minisketch|univalue)/', file_path): + continue + + with open(file_path, 'r', encoding='utf-8') as file_obj: + contents = file_obj.read() + + if OPT_IN_LINE in contents: + continue + + non_comment_pattern = re.compile(r'^\s*((?!#).+)$', re.MULTILINE) + non_comment_lines = re.findall(non_comment_pattern, contents) + if not non_comment_lines: + continue + + first_non_comment_line = non_comment_lines[0] + if first_non_comment_line not in OPT_OUT_LINES: + print(f'Missing "export LC_ALL=C" (to avoid locale dependence) as first non-comment non-empty line in {file_path}') + exit_code = 1 + + return sys.exit(exit_code) + +if __name__ == '__main__': + main() + diff --git a/test/lint/lint-shell-locale.sh b/test/lint/lint-shell-locale.sh deleted file mode 100755 index 4c6b8a57e6..0000000000 --- a/test/lint/lint-shell-locale.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Make sure all shell scripts: -# a.) explicitly opt out of locale dependence using -# "export LC_ALL=C" or "export LC_ALL=C.UTF-8", or -# b.) explicitly opt in to locale dependence using the annotation below. - -export LC_ALL=C - -EXIT_CODE=0 -for SHELL_SCRIPT in $(git ls-files -- "*.sh" | grep -vE "src/(secp256k1|minisketch|univalue)/"); do - if grep -q "# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"" "${SHELL_SCRIPT}"; then - continue - fi - FIRST_NON_COMMENT_LINE=$(grep -vE '^(#.*)?$' "${SHELL_SCRIPT}" | head -1) - if [[ ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C" && ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C.UTF-8" ]]; then - echo "Missing \"export LC_ALL=C\" (to avoid locale dependence) as first non-comment non-empty line in ${SHELL_SCRIPT}" - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-shell.py b/test/lint/lint-shell.py new file mode 100755 index 0000000000..f1e4494350 --- /dev/null +++ b/test/lint/lint-shell.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check for shellcheck warnings in shell scripts. +""" + +import subprocess +import re +import sys + +# Disabled warnings: +DISABLED = [ + 'SC2162', # read without -r will mangle backslashes. +] + +def check_shellcheck_install(): + try: + subprocess.run(['shellcheck', '--version'], stdout=subprocess.DEVNULL, check=True) + except FileNotFoundError: + print('Skipping shell linting since shellcheck is not installed.') + sys.exit(0) + +def get_files(command): + output = subprocess.run(command, stdout=subprocess.PIPE, universal_newlines=True) + files = output.stdout.split('\n') + + # remove whitespace element + files = list(filter(None, files)) + return files + +def main(): + check_shellcheck_install() + + # build the `exclude` flag + exclude = '--exclude=' + ','.join(DISABLED) + + # build the `sourced files` list + sourced_files_cmd = [ + 'git', + 'grep', + '-El', + r'^# shellcheck shell=', + ] + sourced_files = get_files(sourced_files_cmd) + + # build the `guix files` list + guix_files_cmd = [ + 'git', + 'grep', + '-El', + r'^#!\/usr\/bin\/env bash', + '--', + 'contrib/guix', + 'contrib/shell', + ] + guix_files = get_files(guix_files_cmd) + + # build the other script files list + files_cmd = [ + 'git', + 'ls-files', + '--', + '*.sh', + ] + files = get_files(files_cmd) + # remove everything that doesn't match this regex + reg = re.compile(r'src/[leveldb,secp256k1,minisketch,univalue]') + files[:] = [file for file in files if not reg.match(file)] + + # build the `shellcheck` command + shellcheck_cmd = [ + 'shellcheck', + '--external-sources', + '--check-sourced', + '--source-path=SCRIPTDIR', + ] + shellcheck_cmd.append(exclude) + shellcheck_cmd.extend(sourced_files) + shellcheck_cmd.extend(guix_files) + shellcheck_cmd.extend(files) + + # run the `shellcheck` command + try: + subprocess.check_call(shellcheck_cmd) + except subprocess.CalledProcessError: + sys.exit(1) + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh deleted file mode 100755 index 5fa104fce6..0000000000 --- a/test/lint/lint-shell.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for shellcheck warnings in shell scripts. - -export LC_ALL=C - -# Disabled warnings: -disabled=( - SC2162 # read without -r will mangle backslashes. -) - -EXIT_CODE=0 - -if ! command -v shellcheck > /dev/null; then - echo "Skipping shell linting since shellcheck is not installed." - exit $EXIT_CODE -fi - -SHELLCHECK_CMD=(shellcheck --external-sources --check-sourced --source-path=SCRIPTDIR) -EXCLUDE="--exclude=$(IFS=','; echo "${disabled[*]}")" -# Check shellcheck directive used for sourced files -mapfile -t SOURCED_FILES < <(git ls-files | xargs gawk '/^# shellcheck shell=/ {print FILENAME} {nextfile}') -mapfile -t GUIX_FILES < <(git ls-files contrib/guix contrib/shell | xargs gawk '/^#!\/usr\/bin\/env bash/ {print FILENAME} {nextfile}') -mapfile -t FILES < <(git ls-files -- '*.sh' | grep -vE 'src/(leveldb|secp256k1|minisketch|univalue)/') -if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE" "${SOURCED_FILES[@]}" "${GUIX_FILES[@]}" "${FILES[@]}"; then - EXIT_CODE=1 -fi - -exit $EXIT_CODE diff --git a/test/lint/lint-spelling.py b/test/lint/lint-spelling.py new file mode 100755 index 0000000000..5da1b243f7 --- /dev/null +++ b/test/lint/lint-spelling.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Warn in case of spelling errors. +Note: Will exit successfully regardless of spelling errors. +""" + +from subprocess import check_output, STDOUT, CalledProcessError + +IGNORE_WORDS_FILE = 'test/lint/spelling.ignore-words.txt' +FILES_ARGS = ['git', 'ls-files', '--', ":(exclude)build-aux/m4/", ":(exclude)contrib/seeds/*.txt", ":(exclude)depends/", ":(exclude)doc/release-notes/", ":(exclude)src/leveldb/", ":(exclude)src/crc32c/", ":(exclude)src/qt/locale/", ":(exclude)src/qt/*.qrc", ":(exclude)src/secp256k1/", ":(exclude)src/minisketch/", ":(exclude)src/univalue/", ":(exclude)contrib/builder-keys/keys.txt", ":(exclude)contrib/guix/patches"] + + +def check_codespell_install(): + try: + check_output(["codespell", "--version"]) + except FileNotFoundError: + print("Skipping spell check linting since codespell is not installed.") + exit(0) + + +def main(): + check_codespell_install() + + files = check_output(FILES_ARGS).decode("utf-8").splitlines() + codespell_args = ['codespell', '--check-filenames', '--disable-colors', '--quiet-level=7', '--ignore-words={}'.format(IGNORE_WORDS_FILE)] + files + + try: + check_output(codespell_args, stderr=STDOUT) + except CalledProcessError as e: + print(e.output.decode("utf-8"), end="") + print('^ Warning: codespell identified likely spelling errors. Any false positives? Add them to the list of ignored words in {}'.format(IGNORE_WORDS_FILE)) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-spelling.sh b/test/lint/lint-spelling.sh deleted file mode 100755 index b3e558b02a..0000000000 --- a/test/lint/lint-spelling.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Warn in case of spelling errors. -# Note: Will exit successfully regardless of spelling errors. - -export LC_ALL=C - -if ! command -v codespell > /dev/null; then - echo "Skipping spell check linting since codespell is not installed." - exit 0 -fi - -IGNORE_WORDS_FILE=test/lint/lint-spelling.ignore-words.txt -mapfile -t FILES < <(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/qt/locale/" ":(exclude)src/qt/*.qrc" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" ":(exclude)contrib/builder-keys/keys.txt" ":(exclude)contrib/guix/patches") -if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} "${FILES[@]}"; then - echo "^ Warning: codespell identified likely spelling errors. Any false positives? Add them to the list of ignored words in ${IGNORE_WORDS_FILE}" -fi diff --git a/test/lint/lint-submodule.py b/test/lint/lint-submodule.py new file mode 100755 index 0000000000..89d4c80f55 --- /dev/null +++ b/test/lint/lint-submodule.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +This script checks for git modules +""" + +import subprocess +import sys + +def main(): + submodules_list = subprocess.check_output(['git', 'submodule', 'status', '--recursive'], + universal_newlines = True, encoding = 'utf8').rstrip('\n') + if submodules_list: + print("These submodules were found, delete them:\n", submodules_list) + sys.exit(1) + sys.exit(0) + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-submodule.sh b/test/lint/lint-submodule.sh deleted file mode 100755 index d9aa021df7..0000000000 --- a/test/lint/lint-submodule.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# This script checks for git modules -export LC_ALL=C -EXIT_CODE=0 - -CMD=$(git submodule status --recursive) -if test -n "$CMD"; -then - echo These submodules were found, delete them: - echo "$CMD" - EXIT_CODE=1 -fi - -exit $EXIT_CODE - diff --git a/test/lint/lint-tests.py b/test/lint/lint-tests.py new file mode 100755 index 0000000000..849ddcb961 --- /dev/null +++ b/test/lint/lint-tests.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check the test suite naming conventions +""" + +import re +import subprocess +import sys + + +def grep_boost_fixture_test_suite(): + command = [ + "git", + "grep", + "-E", + r"^BOOST_FIXTURE_TEST_SUITE\(", + "--", + "src/test/**.cpp", + "src/wallet/test/**.cpp", + ] + return subprocess.check_output(command, universal_newlines=True, encoding="utf8") + + +def check_matching_test_names(test_suite_list): + not_matching = [ + x + for x in test_suite_list + if re.search(r"/(.*?)\.cpp:BOOST_FIXTURE_TEST_SUITE\(\1, .*\)", x) is None + ] + if len(not_matching) > 0: + not_matching = "\n".join(not_matching) + error_msg = ( + "The test suite in file src/test/foo_tests.cpp should be named\n" + '"foo_tests". Please make sure the following test suites follow\n' + "that convention:\n\n" + f"{not_matching}\n" + ) + print(error_msg) + return 1 + return 0 + + +def get_duplicates(input_list): + """ + From https://stackoverflow.com/a/9835819 + """ + seen = set() + dupes = set() + for x in input_list: + if x in seen: + dupes.add(x) + else: + seen.add(x) + return dupes + + +def check_unique_test_names(test_suite_list): + output = [re.search(r"\((.*?),", x) for x in test_suite_list] + output = [x.group(1) for x in output if x is not None] + output = get_duplicates(output) + output = sorted(list(output)) + + if len(output) > 0: + output = "\n".join(output) + error_msg = ( + "Test suite names must be unique. The following test suite names\n" + f"appear to be used more than once:\n\n{output}" + ) + print(error_msg) + return 1 + return 0 + + +def main(): + test_suite_list = grep_boost_fixture_test_suite().splitlines() + exit_code = check_matching_test_names(test_suite_list) + exit_code |= check_unique_test_names(test_suite_list) + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-tests.sh b/test/lint/lint-tests.sh deleted file mode 100755 index 35d11023eb..0000000000 --- a/test/lint/lint-tests.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check the test suite naming conventions - -export LC_ALL=C -EXIT_CODE=0 - -NAMING_INCONSISTENCIES=$(git grep -E '^BOOST_FIXTURE_TEST_SUITE\(' -- \ - "src/test/**.cpp" "src/wallet/test/**.cpp" | \ - grep -vE '/(.*?)\.cpp:BOOST_FIXTURE_TEST_SUITE\(\1, .*\)$') -if [[ ${NAMING_INCONSISTENCIES} != "" ]]; then - echo "The test suite in file src/test/foo_tests.cpp should be named" - echo "\"foo_tests\". Please make sure the following test suites follow" - echo "that convention:" - echo - echo "${NAMING_INCONSISTENCIES}" - EXIT_CODE=1 -fi - -TEST_SUITE_NAME_COLLISSIONS=$(git grep -E '^BOOST_FIXTURE_TEST_SUITE\(' -- \ - "src/test/**.cpp" "src/wallet/test/**.cpp" | cut -f2 -d'(' | cut -f1 -d, | \ - sort | uniq -d) -if [[ ${TEST_SUITE_NAME_COLLISSIONS} != "" ]]; then - echo "Test suite names must be unique. The following test suite names" - echo "appear to be used more than once:" - echo - echo "${TEST_SUITE_NAME_COLLISSIONS}" - EXIT_CODE=1 -fi - -exit ${EXIT_CODE} diff --git a/test/lint/lint-whitespace.py b/test/lint/lint-whitespace.py new file mode 100755 index 0000000000..d98fc8d9a2 --- /dev/null +++ b/test/lint/lint-whitespace.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2017-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for new lines in diff that introduce trailing whitespace or +# tab characters instead of spaces. + +# We can't run this check unless we know the commit range for the PR. + +import argparse +import os +import re +import sys + +from subprocess import check_output + +EXCLUDED_DIRS = ["depends/patches/", + "contrib/guix/patches/", + "src/leveldb/", + "src/crc32c/", + "src/secp256k1/", + "src/minisketch/", + "src/univalue/", + "doc/release-notes/", + "src/qt/locale"] + +def parse_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description=""" + Check for new lines in diff that introduce trailing whitespace + or tab characters instead of spaces in unstaged changes, the + previous n commits, or a commit-range. + """, + epilog=f""" + You can manually set the commit-range with the COMMIT_RANGE + environment variable (e.g. "COMMIT_RANGE='47ba2c3...ee50c9e' + {sys.argv[0]}"). Defaults to current merge base when neither + prev-commits nor the environment variable is set. + """) + + parser.add_argument("--prev-commits", "-p", required=False, help="The previous n commits to check") + + return parser.parse_args() + + +def report_diff(selection): + filename = "" + seen = False + seenln = False + + print("The following changes were suspected:") + + for line in selection: + if re.match(r"^diff", line): + filename = line + seen = False + elif re.match(r"^@@", line): + linenumber = line + seenln = False + else: + if not seen: + # The first time a file is seen with trailing whitespace or a tab character, we print the + # filename (preceded by a newline). + print("") + print(filename) + seen = True + if not seenln: + print(linenumber) + seenln = True + print(line) + + +def get_diff(commit_range, check_only_code): + exclude_args = [":(exclude)" + dir for dir in EXCLUDED_DIRS] + + if check_only_code: + what_files = ["*.cpp", "*.h", "*.md", "*.py", "*.sh"] + else: + what_files = ["."] + + diff = check_output(["git", "diff", "-U0", commit_range, "--"] + what_files + exclude_args, universal_newlines=True, encoding="utf8") + + return diff + + +def main(): + args = parse_args() + + if not os.getenv("COMMIT_RANGE"): + if args.prev_commits: + commit_range = "HEAD~" + args.prev_commits + "...HEAD" + else: + # This assumes that the target branch of the pull request will be master. + merge_base = check_output(["git", "merge-base", "HEAD", "master"], universal_newlines=True, encoding="utf8").rstrip("\n") + commit_range = merge_base + "..HEAD" + else: + commit_range = os.getenv("COMMIT_RANGE") + + whitespace_selection = [] + tab_selection = [] + + # Check if trailing whitespace was found in the diff. + for line in get_diff(commit_range, check_only_code=False).splitlines(): + if re.match(r"^(diff --git|\@@|^\+.*\s+$)", line): + whitespace_selection.append(line) + + whitespace_additions = [i for i in whitespace_selection if i.startswith("+")] + + # Check if tab characters were found in the diff. + for line in get_diff(commit_range, check_only_code=True).splitlines(): + if re.match(r"^(diff --git|\@@|^\+.*\t)", line): + tab_selection.append(line) + + tab_additions = [i for i in tab_selection if i.startswith("+")] + + ret = 0 + + if len(whitespace_additions) > 0: + print("This diff appears to have added new lines with trailing whitespace.") + report_diff(whitespace_selection) + ret = 1 + + if len(tab_additions) > 0: + print("This diff appears to have added new lines with tab characters instead of spaces.") + report_diff(tab_selection) + ret = 1 + + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-whitespace.sh b/test/lint/lint-whitespace.sh deleted file mode 100755 index 9d55c71eb5..0000000000 --- a/test/lint/lint-whitespace.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2017-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for new lines in diff that introduce trailing whitespace. - -# We can't run this check unless we know the commit range for the PR. - -export LC_ALL=C -while getopts "?" opt; do - case $opt in - ?) - echo "Usage: $0 [N]" - echo " COMMIT_RANGE='<commit range>' $0" - echo " $0 -?" - echo "Checks unstaged changes, the previous N commits, or a commit range." - echo "COMMIT_RANGE='47ba2c3...ee50c9e' $0" - exit 0 - ;; - esac -done - -if [ -z "${COMMIT_RANGE}" ]; then - if [ -n "$1" ]; then - COMMIT_RANGE="HEAD~$1...HEAD" - else - # This assumes that the target branch of the pull request will be master. - MERGE_BASE=$(git merge-base HEAD master) - COMMIT_RANGE="$MERGE_BASE..HEAD" - fi -fi - -showdiff() { - if ! git diff -U0 "${COMMIT_RANGE}" -- "." ":(exclude)depends/patches/" ":(exclude)contrib/guix/patches/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/" ":(exclude)src/qt/locale/"; then - echo "Failed to get a diff" - exit 1 - fi -} - -showcodediff() { - if ! git diff -U0 "${COMMIT_RANGE}" -- *.cpp *.h *.md *.py *.sh ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/" ":(exclude)src/qt/locale/"; then - echo "Failed to get a diff" - exit 1 - fi -} - -RET=0 - -# Check if trailing whitespace was found in the diff. -if showdiff | grep -E -q '^\+.*\s+$'; then - echo "This diff appears to have added new lines with trailing whitespace." - echo "The following changes were suspected:" - FILENAME="" - SEEN=0 - SEENLN=0 - while read -r line; do - if [[ "$line" =~ ^diff ]]; then - FILENAME="$line" - SEEN=0 - elif [[ "$line" =~ ^@@ ]]; then - LINENUMBER="$line" - SEENLN=0 - else - if [ "$SEEN" -eq 0 ]; then - # The first time a file is seen with trailing whitespace, we print the - # filename (preceded by a newline). - echo - echo "$FILENAME" - SEEN=1 - fi - if [ "$SEENLN" -eq 0 ]; then - echo "$LINENUMBER" - SEENLN=1 - fi - echo "$line" - fi - done < <(showdiff | grep -E '^(diff --git |@@|\+.*\s+$)') - RET=1 -fi - -# Check if tab characters were found in the diff. -if showcodediff | perl -nle '$MATCH++ if m{^\+.*\t}; END{exit 1 unless $MATCH>0}' > /dev/null; then - echo "This diff appears to have added new lines with tab characters instead of spaces." - echo "The following changes were suspected:" - FILENAME="" - SEEN=0 - SEENLN=0 - while read -r line; do - if [[ "$line" =~ ^diff ]]; then - FILENAME="$line" - SEEN=0 - elif [[ "$line" =~ ^@@ ]]; then - LINENUMBER="$line" - SEENLN=0 - else - if [ "$SEEN" -eq 0 ]; then - # The first time a file is seen with a tab character, we print the - # filename (preceded by a newline). - echo - echo "$FILENAME" - SEEN=1 - fi - if [ "$SEENLN" -eq 0 ]; then - echo "$LINENUMBER" - SEENLN=1 - fi - echo "$line" - fi - done < <(showcodediff | perl -nle 'print if m{^(diff --git |@@|\+.*\t)}') - RET=1 -fi - -exit $RET diff --git a/test/lint/run-lint-format-strings.py b/test/lint/run-lint-format-strings.py new file mode 100755 index 0000000000..b814446125 --- /dev/null +++ b/test/lint/run-lint-format-strings.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Lint format strings: This program checks that the number of arguments passed +# to a variadic format string function matches the number of format specifiers +# in the format string. + +import argparse +import re +import sys + +FALSE_POSITIVES = [ + ("src/dbwrapper.cpp", "vsnprintf(p, limit - p, format, backup_ap)"), + ("src/index/base.cpp", "FatalError(const char* fmt, const Args&... args)"), + ("src/netbase.cpp", "LogConnectFailure(bool manual_connection, const char* fmt, const Args&... args)"), + ("src/clientversion.cpp", "strprintf(_(COPYRIGHT_HOLDERS).translated, COPYRIGHT_HOLDERS_SUBSTITUTION)"), + ("src/validationinterface.cpp", "LogPrint(BCLog::VALIDATION, fmt \"\\n\", __VA_ARGS__)"), + ("src/wallet/wallet.h", "WalletLogPrintf(std::string fmt, Params... parameters)"), + ("src/wallet/wallet.h", "LogPrintf((\"%s \" + fmt).c_str(), GetDisplayName(), parameters...)"), + ("src/wallet/scriptpubkeyman.h", "WalletLogPrintf(std::string fmt, Params... parameters)"), + ("src/wallet/scriptpubkeyman.h", "LogPrintf((\"%s \" + fmt).c_str(), m_storage.GetDisplayName(), parameters...)"), +] + + +def parse_function_calls(function_name, source_code): + """Return an array with all calls to function function_name in string source_code. + Preprocessor directives and C++ style comments ("//") in source_code are removed. + + >>> len(parse_function_calls("foo", "foo();bar();foo();bar();")) + 2 + >>> parse_function_calls("foo", "foo(1);bar(1);foo(2);bar(2);")[0].startswith("foo(1);") + True + >>> parse_function_calls("foo", "foo(1);bar(1);foo(2);bar(2);")[1].startswith("foo(2);") + True + >>> len(parse_function_calls("foo", "foo();bar();// foo();bar();")) + 1 + >>> len(parse_function_calls("foo", "#define FOO foo();")) + 0 + """ + assert type(function_name) is str and type(source_code) is str and function_name + lines = [re.sub("// .*", " ", line).strip() + for line in source_code.split("\n") + if not line.strip().startswith("#")] + return re.findall(r"[^a-zA-Z_](?=({}\(.*).*)".format(function_name), " " + " ".join(lines)) + + +def normalize(s): + """Return a normalized version of string s with newlines, tabs and C style comments ("/* ... */") + replaced with spaces. Multiple spaces are replaced with a single space. + + >>> normalize(" /* nothing */ foo\tfoo /* bar */ foo ") + 'foo foo foo' + """ + assert type(s) is str + s = s.replace("\n", " ") + s = s.replace("\t", " ") + s = re.sub(r"/\*.*?\*/", " ", s) + s = re.sub(" {2,}", " ", s) + return s.strip() + + +ESCAPE_MAP = { + r"\n": "[escaped-newline]", + r"\t": "[escaped-tab]", + r'\"': "[escaped-quote]", +} + + +def escape(s): + """Return the escaped version of string s with "\\\"", "\\n" and "\\t" escaped as + "[escaped-backslash]", "[escaped-newline]" and "[escaped-tab]". + + >>> unescape(escape("foo")) == "foo" + True + >>> escape(r'foo \\t foo \\n foo \\\\ foo \\ foo \\"bar\\"') + 'foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]' + """ + assert type(s) is str + for raw_value, escaped_value in ESCAPE_MAP.items(): + s = s.replace(raw_value, escaped_value) + return s + + +def unescape(s): + """Return the unescaped version of escaped string s. + Reverses the replacements made in function escape(s). + + >>> unescape(escape("bar")) + 'bar' + >>> unescape("foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]") + 'foo \\\\t foo \\\\n foo \\\\\\\\ foo \\\\ foo \\\\"bar\\\\"' + """ + assert type(s) is str + for raw_value, escaped_value in ESCAPE_MAP.items(): + s = s.replace(escaped_value, raw_value) + return s + + +def parse_function_call_and_arguments(function_name, function_call): + """Split string function_call into an array of strings consisting of: + * the string function_call followed by "(" + * the function call argument #1 + * ... + * the function call argument #n + * a trailing ");" + + The strings returned are in escaped form. See escape(...). + + >>> parse_function_call_and_arguments("foo", 'foo("%s", "foo");') + ['foo(', '"%s",', ' "foo"', ')'] + >>> parse_function_call_and_arguments("foo", 'foo("%s", "foo");') + ['foo(', '"%s",', ' "foo"', ')'] + >>> parse_function_call_and_arguments("foo", 'foo("%s %s", "foo", "bar");') + ['foo(', '"%s %s",', ' "foo",', ' "bar"', ')'] + >>> parse_function_call_and_arguments("fooprintf", 'fooprintf("%050d", i);') + ['fooprintf(', '"%050d",', ' i', ')'] + >>> parse_function_call_and_arguments("foo", 'foo(bar(foobar(barfoo("foo"))), foobar); barfoo') + ['foo(', 'bar(foobar(barfoo("foo"))),', ' foobar', ')'] + >>> parse_function_call_and_arguments("foo", "foo()") + ['foo(', '', ')'] + >>> parse_function_call_and_arguments("foo", "foo(123)") + ['foo(', '123', ')'] + >>> parse_function_call_and_arguments("foo", 'foo("foo")') + ['foo(', '"foo"', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>,wchar_t>().to_bytes(buf), err);') + ['strprintf(', '"%s (%d)",', ' std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>,wchar_t>().to_bytes(buf),', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo<wchar_t>().to_bytes(buf), err);') + ['strprintf(', '"%s (%d)",', ' foo<wchar_t>().to_bytes(buf),', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo().to_bytes(buf), err);') + ['strprintf(', '"%s (%d)",', ' foo().to_bytes(buf),', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo << 1, err);') + ['strprintf(', '"%s (%d)",', ' foo << 1,', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo<bar>() >> 1, err);') + ['strprintf(', '"%s (%d)",', ' foo<bar>() >> 1,', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo < 1 ? bar : foobar, err);') + ['strprintf(', '"%s (%d)",', ' foo < 1 ? bar : foobar,', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo < 1, err);') + ['strprintf(', '"%s (%d)",', ' foo < 1,', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo > 1 ? bar : foobar, err);') + ['strprintf(', '"%s (%d)",', ' foo > 1 ? bar : foobar,', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo > 1, err);') + ['strprintf(', '"%s (%d)",', ' foo > 1,', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo <= 1, err);') + ['strprintf(', '"%s (%d)",', ' foo <= 1,', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo <= bar<1, 2>(1, 2), err);') + ['strprintf(', '"%s (%d)",', ' foo <= bar<1, 2>(1, 2),', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo>foo<1,2>(1,2)?bar:foobar,err)'); + ['strprintf(', '"%s (%d)",', ' foo>foo<1,2>(1,2)?bar:foobar,', 'err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo>foo<1,2>(1,2),err)'); + ['strprintf(', '"%s (%d)",', ' foo>foo<1,2>(1,2),', 'err', ')'] + """ + assert type(function_name) is str and type(function_call) is str and function_name + remaining = normalize(escape(function_call)) + expected_function_call = "{}(".format(function_name) + assert remaining.startswith(expected_function_call) + parts = [expected_function_call] + remaining = remaining[len(expected_function_call):] + open_parentheses = 1 + open_template_arguments = 0 + in_string = False + parts.append("") + for i, char in enumerate(remaining): + parts.append(parts.pop() + char) + if char == "\"": + in_string = not in_string + continue + if in_string: + continue + if char == "(": + open_parentheses += 1 + continue + if char == ")": + open_parentheses -= 1 + if open_parentheses > 1: + continue + if open_parentheses == 0: + parts.append(parts.pop()[:-1]) + parts.append(char) + break + prev_char = remaining[i - 1] if i - 1 >= 0 else None + next_char = remaining[i + 1] if i + 1 <= len(remaining) - 1 else None + if char == "<" and next_char not in [" ", "<", "="] and prev_char not in [" ", "<"]: + open_template_arguments += 1 + continue + if char == ">" and next_char not in [" ", ">", "="] and prev_char not in [" ", ">"] and open_template_arguments > 0: + open_template_arguments -= 1 + if open_template_arguments > 0: + continue + if char == ",": + parts.append("") + return parts + + +def parse_string_content(argument): + """Return the text within quotes in string argument. + + >>> parse_string_content('1 "foo %d bar" 2') + 'foo %d bar' + >>> parse_string_content('1 foobar 2') + '' + >>> parse_string_content('1 "bar" 2') + 'bar' + >>> parse_string_content('1 "foo" 2 "bar" 3') + 'foobar' + >>> parse_string_content('1 "foo" 2 " " "bar" 3') + 'foo bar' + >>> parse_string_content('""') + '' + >>> parse_string_content('') + '' + >>> parse_string_content('1 2 3') + '' + """ + assert type(argument) is str + string_content = "" + in_string = False + for char in normalize(escape(argument)): + if char == "\"": + in_string = not in_string + elif in_string: + string_content += char + return string_content + + +def count_format_specifiers(format_string): + """Return the number of format specifiers in string format_string. + + >>> count_format_specifiers("foo bar foo") + 0 + >>> count_format_specifiers("foo %d bar foo") + 1 + >>> count_format_specifiers("foo %d bar %i foo") + 2 + >>> count_format_specifiers("foo %d bar %i foo %% foo") + 2 + >>> count_format_specifiers("foo %d bar %i foo %% foo %d foo") + 3 + >>> count_format_specifiers("foo %d bar %i foo %% foo %*d foo") + 4 + """ + assert type(format_string) is str + format_string = format_string.replace('%%', 'X') + n = 0 + in_specifier = False + for i, char in enumerate(format_string): + if char == "%": + in_specifier = True + n += 1 + elif char in "aAcdeEfFgGinopsuxX": + in_specifier = False + elif in_specifier and char == "*": + n += 1 + return n + + +def main(): + parser = argparse.ArgumentParser(description="This program checks that the number of arguments passed " + "to a variadic format string function matches the number of format " + "specifiers in the format string.") + parser.add_argument("--skip-arguments", type=int, help="number of arguments before the format string " + "argument (e.g. 1 in the case of fprintf)", default=0) + parser.add_argument("function_name", help="function name (e.g. fprintf)", default=None) + parser.add_argument("file", nargs="*", help="C++ source code file (e.g. foo.cpp)") + args = parser.parse_args() + exit_code = 0 + for filename in args.file: + with open(filename, "r", encoding="utf-8") as f: + for function_call_str in parse_function_calls(args.function_name, f.read()): + parts = parse_function_call_and_arguments(args.function_name, function_call_str) + relevant_function_call_str = unescape("".join(parts))[:512] + if (f.name, relevant_function_call_str) in FALSE_POSITIVES: + continue + if len(parts) < 3 + args.skip_arguments: + exit_code = 1 + print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str)) + continue + argument_count = len(parts) - 3 - args.skip_arguments + format_str = parse_string_content(parts[1 + args.skip_arguments]) + format_specifier_count = count_format_specifiers(format_str) + if format_specifier_count != argument_count: + exit_code = 1 + print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str)) + continue + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-spelling.ignore-words.txt b/test/lint/spelling.ignore-words.txt index afdb0692d8..afdb0692d8 100644 --- a/test/lint/lint-spelling.ignore-words.txt +++ b/test/lint/spelling.ignore-words.txt diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index b06dd253be..e6cfe5f81a 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -62,7 +62,6 @@ implicit-integer-sign-change:script/bitcoinconsensus.cpp implicit-integer-sign-change:script/interpreter.cpp implicit-integer-sign-change:serialize.h implicit-integer-sign-change:txmempool.cpp -implicit-signed-integer-truncation:addrman.cpp implicit-signed-integer-truncation:crypto/ implicit-unsigned-integer-truncation:crypto/ shift-base:arith_uint256.cpp diff --git a/test/util/test_runner.py b/test/util/test_runner.py index a7fc3b1dc1..03db05c563 100755 --- a/test/util/test_runner.py +++ b/test/util/test_runner.py @@ -22,7 +22,8 @@ import sys def main(): config = configparser.ConfigParser() config.optionxform = str - config.read_file(open(os.path.join(os.path.dirname(__file__), "../config.ini"), encoding="utf8")) + with open(os.path.join(os.path.dirname(__file__), "../config.ini"), encoding="utf8") as f: + config.read_file(f) env_conf = dict(config.items('environment')) parser = argparse.ArgumentParser(description=__doc__) @@ -43,7 +44,8 @@ def main(): def bctester(testDir, input_basename, buildenv): """ Loads and parses the input file, runs all tests and reports results""" input_filename = os.path.join(testDir, input_basename) - raw_data = open(input_filename, encoding="utf8").read() + with open(input_filename, encoding="utf8") as f: + raw_data = f.read() input_data = json.loads(raw_data) failed_testcases = [] @@ -80,7 +82,8 @@ def bctest(testDir, testObj, buildenv): inputData = None if "input" in testObj: filename = os.path.join(testDir, testObj["input"]) - inputData = open(filename, encoding="utf8").read() + with open(filename, encoding="utf8") as f: + inputData = f.read() stdinCfg = subprocess.PIPE # Read the expected output data (if there is any) @@ -91,7 +94,8 @@ def bctest(testDir, testObj, buildenv): outputFn = testObj['output_cmp'] outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare) try: - outputData = open(os.path.join(testDir, outputFn), encoding="utf8").read() + with open(os.path.join(testDir, outputFn), encoding="utf8") as f: + outputData = f.read() except: logging.error("Output file " + outputFn + " cannot be opened") raise |