diff options
244 files changed, 7959 insertions, 3665 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index e07ff9796e..6cf91f8752 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -2,6 +2,7 @@ env: # Global defaults PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y" MAKEJOBS: "-j10" TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache + CI_FAILFAST_TEST_LEAVE_DANGLING: "1" # Cirrus CI does not care about dangling process and setting this variable avoids killing the CI script itself on error CCACHE_SIZE: "200M" CCACHE_DIR: "/tmp/ccache_dir" CCACHE_NOHASHDIR: "1" # Debug info might contain a stale path if the build dir changes, but this is fine @@ -85,9 +86,11 @@ task: CI_VCPKG_TAG: '2022.02.23' VCPKG_DOWNLOADS: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\downloads' VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' - QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.2/single/qt-everywhere-src-5.15.2.zip' - QT_LOCAL_PATH: 'C:\qt-everywhere-src-5.15.2.zip' - QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.2' + CCACHE_DIR: 'C:\Users\ContainerAdministrator\AppData\Local\ccache' + WRAPPED_CL: 'C:\Users\ContainerAdministrator\AppData\Local\Temp\cirrus-ci-build\ci\test\wrapped-cl.bat' + QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.3/single/qt-everywhere-opensource-src-5.15.3.zip' + QT_LOCAL_PATH: 'C:\qt-everywhere-opensource-src-5.15.3.zip' + QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.3' QTBASEDIR: 'C:\Qt_static' x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"' IgnoreWarnIntDirInTempDetected: 'true' @@ -134,9 +137,13 @@ task: - msbuild -version populate_script: - mkdir %VCPKG_DEFAULT_BINARY_CACHE% - install_python_script: + ccache_cache: + folder: '%CCACHE_DIR%' + install_tools_script: + - choco install --yes --no-progress ccache - choco install --yes --no-progress python3 --version=3.9.6 - pip install zmq + - ccache --version - python -VV install_vcpkg_script: - cd .. @@ -148,9 +155,12 @@ task: - .\vcpkg integrate install - .\vcpkg version build_script: + - '%x64_NATIVE_TOOLS%' - cd %CIRRUS_WORKING_DIR% + - ccache --zero-stats - python build_msvc\msvc-autogen.py - - msbuild build_msvc\bitcoin.sln -property:Configuration=Release -maxCpuCount -verbosity:minimal -noLogo + - msbuild build_msvc\bitcoin.sln -property:CLToolExe=%WRAPPED_CL% -property:Configuration=Release -maxCpuCount -verbosity:minimal -noLogo + - ccache --show-stats unit_tests_script: - src\test_bitcoin.exe -l test_suite - src\bench_bitcoin.exe > NUL @@ -288,7 +298,7 @@ task: << : *GLOBAL_TASK_TEMPLATE macos_instance: # Use latest image, but hardcode version to avoid silent upgrades (and breaks) - image: monterey-xcode-13.2 # https://cirrus-ci.org/guide/macOS + image: monterey-xcode-13.3 # https://cirrus-ci.org/guide/macOS env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV CI_USE_APT_INSTALL: "no" diff --git a/.gitignore b/.gitignore index 0f07a86401..cde517c986 100644 --- a/.gitignore +++ b/.gitignore @@ -151,3 +151,5 @@ osx_volname dist/ /guix-build-* + +/ci/scratch/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 254e610393..ba78a20ae3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -153,7 +153,8 @@ the pull request affects. Valid areas as: - `test`, `qa` or `ci` for changes to the unit tests, QA tests or CI code - `util` or `lib` for changes to the utils or libraries - `wallet` for changes to the wallet code - - `build` for changes to the GNU Autotools or reproducible builds + - `build` for changes to the GNU Autotools or MSVC builds + - `guix` for changes to the GUIX reproducible builds Examples: diff --git a/Makefile.am b/Makefile.am index 1412624d54..8637af362e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -64,7 +64,6 @@ WINDOWS_PACKAGING = $(top_srcdir)/share/pixmaps/bitcoin.ico \ $(top_srcdir)/doc/README_windows.txt OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_INSTALLER_ICONS) \ - $(top_srcdir)/contrib/macdeploy/detached-sig-apply.sh \ $(top_srcdir)/contrib/macdeploy/detached-sig-create.sh COVERAGE_INFO = $(COV_TOOL_WRAPPER) baseline.info \ diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index 1454e33f6e..a716cd9a27 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -116,8 +116,8 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ BITCOIN_QT_CHECK([ TEMP_CPPFLAGS=$CPPFLAGS TEMP_CXXFLAGS=$CXXFLAGS - CPPFLAGS="$QT_INCLUDES $CPPFLAGS" - CXXFLAGS="$PIC_FLAGS $CXXFLAGS" + CPPFLAGS="$QT_INCLUDES $CORE_CPPFLAGS $CPPFLAGS" + CXXFLAGS="$PIC_FLAGS $CORE_CXXFLAGS $CXXFLAGS" _BITCOIN_QT_IS_STATIC if test "$bitcoin_cv_static_qt" = "yes"; then _BITCOIN_QT_CHECK_STATIC_LIBS @@ -178,8 +178,8 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ AC_MSG_CHECKING([whether -fPIE can be used with this Qt config]) TEMP_CPPFLAGS=$CPPFLAGS TEMP_CXXFLAGS=$CXXFLAGS - CPPFLAGS="$QT_INCLUDES $CPPFLAGS" - CXXFLAGS="$PIE_FLAGS $CXXFLAGS" + CPPFLAGS="$QT_INCLUDES $CORE_CPPFLAGS $CPPFLAGS" + CXXFLAGS="$PIE_FLAGS $CORE_CXXFLAGS $CXXFLAGS" AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include <QtCore/qconfig.h> #ifndef QT_VERSION @@ -201,7 +201,7 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ BITCOIN_QT_CHECK([ AC_MSG_CHECKING([whether -fPIC is needed with this Qt config]) TEMP_CPPFLAGS=$CPPFLAGS - CPPFLAGS="$QT_INCLUDES $CPPFLAGS" + CPPFLAGS="$QT_INCLUDES $CORE_CPPFLAGS $CPPFLAGS" AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include <QtCore/qconfig.h> #ifndef QT_VERSION diff --git a/build_msvc/README.md b/build_msvc/README.md index cabe4d55ec..7feee6b766 100644 --- a/build_msvc/README.md +++ b/build_msvc/README.md @@ -28,7 +28,7 @@ Qt --------------------- To build Bitcoin Core with the GUI, a static build of Qt is required. -1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-src-5.15.2.zip`](https://download.qt.io/official_releases/qt/5.15/5.15.2/single/qt-everywhere-src-5.15.2.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`. +1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-opensource-src-5.15.3.zip`](https://download.qt.io/official_releases/qt/5.15/5.15.3/single/qt-everywhere-opensource-src-5.15.3.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`. 2. Open "x64 Native Tools Command Prompt for VS 2019", and input the following commands: ```cmd @@ -83,4 +83,4 @@ If is it enabled then in the output `Dynamic base` will be listed in the `DLL ch Terminal Server Aware ``` -This may not disable all stack randomization as versions of windows employ additional stack randomization protections. These protections must be turned off in the OS configuration.
\ No newline at end of file +This may not disable all stack randomization as versions of windows employ additional stack randomization protections. These protections must be turned off in the OS configuration. diff --git a/build_msvc/bitcoin_config.h.in b/build_msvc/bitcoin_config.h.in index e25024e871..b37d536947 100644 --- a/build_msvc/bitcoin_config.h.in +++ b/build_msvc/bitcoin_config.h.in @@ -125,10 +125,6 @@ don't. */ #define HAVE_DECL_STRERROR_R 0 -/* Define to 1 if you have the declaration of `strnlen', and to 0 if you - don't. */ -#define HAVE_DECL_STRNLEN 1 - /* Define if the dllexport attribute is supported. */ #define HAVE_DLLEXPORT_ATTRIBUTE 1 diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh index b333635759..766424769d 100755 --- a/ci/test/00_setup_env_i686_multiprocess.sh +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -15,4 +15,3 @@ export GOAL="install" export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' LDFLAGS='--rtlib=compiler-rt -lgcc_s'" export TEST_RUNNER_ENV="BITCOIND=bitcoin-node" export TEST_RUNNER_EXTRA="--nosandbox" -export PIP_PACKAGES="lief" diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index b03a7edb54..69883e3609 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -11,4 +11,4 @@ export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent- export DOCKER_NAME_TAG=ubuntu:22.04 export NO_DEPENDS=1 export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" +export BITCOIN_CONFIG="--enable-c++20 --enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 619acc4677..071bac8fb3 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -13,10 +13,11 @@ LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_ export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_msan" -export PACKAGES="clang-9 llvm-9 cmake" +export PACKAGES="clang-12 llvm-12 cmake" +# BDB generates false-positives and will be removed in future export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" export RUN_UNIT_TESTS="false" export RUN_FUNCTIONAL_TESTS="false" diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index a7715a6ded..9477fb2d9f 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -15,5 +15,6 @@ export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export FUZZ_TESTS_CONFIG="--valgrind" export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++" +# Temporarily pin dwarf 4, until valgrind can understand clang's dwarf 5 +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++ CXXFLAGS='-fdebug-default-version=4'" export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index b8418106c6..34a792ec8f 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -11,13 +11,13 @@ LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" -export BDB_PREFIX="${BASE_ROOT_DIR}/db4" export CONTAINER_NAME="ci_native_msan" -export PACKAGES="clang-9 llvm-9 cmake" +export PACKAGES="clang-12 llvm-12 cmake" +# BDB generates false-positives and will be removed in future export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--enable-wallet --with-sanitizers=memory --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' BDB_LIBS='-L${BDB_PREFIX}/lib -ldb_cxx-4.8' BDB_CFLAGS='-I${BDB_PREFIX}/include'" +export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" export RUN_FUNCTIONAL_TESTS="false" export CCACHE_SIZE=250M diff --git a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh index 89d6256298..763a5e8b6f 100755 --- a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh +++ b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel export DOCKER_NAME_TAG=ubuntu:18.04 # Use bionic to have one config run the tests in python3.6, see doc/dependencies.md -export PACKAGES="python3-zmq clang-7 llvm-7 libc++abi-7-dev libc++-7-dev" # Use clang-7 to test C++17 compatibility, see doc/dependencies.md -export DEP_OPTS="NO_WALLET=1 CC=clang-7 CXX='clang++-7 -stdlib=libc++'" +export PACKAGES="python3-zmq clang-8 llvm-8 libc++abi-8-dev libc++-8-dev" # Use clang-8 to test C++17 compatibility, see doc/dependencies.md +export DEP_OPTS="NO_WALLET=1 CC=clang-8 CXX='clang++-8 -stdlib=libc++'" export GOAL="install" -export BITCOIN_CONFIG="--enable-reduce-exports CC=clang-7 CXX='clang++-7 -stdlib=libc++' --enable-experimental-util-chainstate" +export BITCOIN_CONFIG="--enable-reduce-exports CC=clang-8 CXX='clang++-8 -stdlib=libc++' --enable-experimental-util-chainstate" diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 2424a11357..7fb5186a87 100755 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -16,4 +16,4 @@ export RUN_UNIT_TESTS="false" export GOAL="install" export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.14.3 v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1 v0.20.1 v0.21.0 v22.0" export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-reduce-exports \ ---enable-debug --disable-fuzz-binary CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" CC=gcc-8 CXX=g++-8" +--enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" CC=gcc-8 CXX=g++-8" diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 0036255caf..ae942d892b 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan export DOCKER_NAME_TAG=ubuntu:22.04 -export PACKAGES="clang llvm libc++abi-dev libc++-dev python3-zmq" -export DEP_OPTS="CC=clang CXX='clang++ -stdlib=libc++'" +export PACKAGES="clang-13 llvm-13 libc++abi-13-dev libc++-13-dev python3-zmq" +export DEP_OPTS="CC=clang-13 CXX='clang++-13 -stdlib=libc++'" export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' CXXFLAGS='-g' --with-sanitizers=thread CC=clang CXX='clang++ -stdlib=libc++'" +export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' CXXFLAGS='-g' --with-sanitizers=thread CC=clang-13 CXX='clang++-13 -stdlib=libc++'" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 646070a84e..7b714dff5c 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -13,4 +13,5 @@ export USE_VALGRIND=1 export NO_DEPENDS=1 export TEST_RUNNER_EXTRA="--nosandbox --exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++" # TODO enable GUI +# Temporarily pin dwarf 4, until valgrind can understand clang's dwarf 5 +export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++ CXXFLAGS='-fdebug-default-version=4'" # TODO enable GUI diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index e409df62eb..c1324a0b14 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -103,11 +103,11 @@ fi CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - CI_EXEC "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-9) 100" - CI_EXEC "update-alternatives --install /usr/bin/clang clang \$(which clang-9) 100" + CI_EXEC "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-12) 100" + CI_EXEC "update-alternatives --install /usr/bin/clang clang \$(which clang-12) 100" CI_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/" CI_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-12.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project" - CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=Memory -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/" + CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/" CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx" fi diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index 8f75fbd1fa..d8dcb708aa 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -36,13 +36,6 @@ if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then CI_EXEC "yes | ${ANDROID_HOME}/cmdline-tools/tools/bin/sdkmanager --install \"build-tools;${ANDROID_BUILD_TOOLS_VERSION}\" \"platform-tools\" \"platforms;android-${ANDROID_API_LEVEL}\" \"ndk;${ANDROID_NDK_VERSION}\"" fi -if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - # Use BDB compiled using install_db4.sh script to work around linking issue when using BDB - # from depends. See https://github.com/bitcoin/bitcoin/pull/18288#discussion_r433189350 for - # details. - CI_EXEC "contrib/install_db4.sh \$(pwd) --enable-umrw CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" -fi - if [ -z "$NO_DEPENDS" ]; then if [[ $DOCKER_NAME_TAG == *centos* ]]; then # CentOS has problems building the depends if the config shell is not explicitly set diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index de42aa6eb1..e70d811d5a 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -27,7 +27,7 @@ if [ "$RUN_UNIT_TESTS" = "true" ]; then fi if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then - CI_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${BASE_BUILD_DIR}/bitcoin-*/src/test/test_bitcoin*" --catch_system_errors=no -l test_suite + CI_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${BASE_OUTDIR}/bin/test_bitcoin" --catch_system_errors=no -l test_suite fi if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then diff --git a/ci/test/wrapped-cl.bat b/ci/test/wrapped-cl.bat new file mode 100644 index 0000000000..fc2a604c58 --- /dev/null +++ b/ci/test/wrapped-cl.bat @@ -0,0 +1 @@ +ccache cl %* diff --git a/configure.ac b/configure.ac index 06a208835b..c345d590cb 100644 --- a/configure.ac +++ b/configure.ac @@ -42,14 +42,9 @@ AH_TOP([#ifndef BITCOIN_CONFIG_H]) AH_TOP([#define BITCOIN_CONFIG_H]) AH_BOTTOM([#endif //BITCOIN_CONFIG_H]) -dnl faketime breaks configure and is only needed for make. Disable it here. -unset FAKETIME - dnl Automake init set-up and checks AM_INIT_AUTOMAKE([1.13 no-define subdir-objects foreign]) -dnl faketime messes with timestamps and causes configure to be re-run. -dnl --disable-maintainer-mode can be used to bypass this. AM_MAINTAINER_MODE([enable]) dnl make the compilation flags quiet unless V=1 is used @@ -78,8 +73,18 @@ AC_ARG_WITH([seccomp], [seccomp_found=$withval], [seccomp_found=auto]) +AC_ARG_ENABLE([c++20], + [AS_HELP_STRING([--enable-c++20], + [enable compilation in c++20 mode (disabled by default)])], + [use_cxx20=$enableval], + [use_cxx20=no]) + dnl Require C++17 compiler (no GNU extensions) +if test "$use_cxx20" = "no"; then AX_CXX_COMPILE_STDCXX([17], [noext], [mandatory]) +else +AX_CXX_COMPILE_STDCXX([20], [noext], [mandatory]) +fi dnl Check if -latomic is required for <std::atomic> CHECK_ATOMIC @@ -96,10 +101,8 @@ fi AC_PROG_OBJCXX ]) -dnl Since libtool 1.5.2 (released 2004-01-25), on Linux libtool no longer -dnl sets RPATH for any directories in the dynamic linker search path. -dnl See more: https://wiki.debian.org/RpathIssue -LT_PREREQ([1.5.2]) +dnl OpenBSD ships with 2.4.2 +LT_PREREQ([2.4.2]) dnl Libtool init checks. LT_INIT([pic-only win32-dll]) @@ -358,7 +361,9 @@ case $host in esac if test "$enable_debug" = "yes"; then - dnl Clear default -g -O2 flags + dnl If debugging is enabled, and the user hasn't overriden CXXFLAGS, clear + dnl them, to prevent autoconfs "-g -O2" being added. Otherwise we'd end up + dnl with "-O0 -g3 -g -O2". if test "$CXXFLAGS_overridden" = "no"; then CXXFLAGS="" fi @@ -465,7 +470,7 @@ if test "$CXXFLAGS_overridden" = "no"; then fi dnl Don't allow extended (non-ASCII) symbols in identifiers. This is easier for code review. -AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers], [CXXFLAGS="$CXXFLAGS -fno-extended-identifiers"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers], [CORE_CXXFLAGS="$CORE_CXXFLAGS -fno-extended-identifiers"], [], [$CXXFLAG_WERROR]) enable_sse42=no enable_sse41=no @@ -614,7 +619,7 @@ CXXFLAGS="$TEMP_CXXFLAGS" fi -CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO" +CORE_CPPFLAGS="$CORE_CPPFLAGS -DHAVE_BUILD_INFO" AC_ARG_WITH([utils], [AS_HELP_STRING([--with-utils], @@ -696,7 +701,7 @@ case $host in AC_MSG_ERROR([windres not found]) fi - CPPFLAGS="$CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -D_WIN32_WINNT=0x0601 -D_WIN32_IE=0x0501 -DWIN32_LEAN_AND_MEAN" + CORE_CPPFLAGS="$CORE_CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -D_WIN32_WINNT=0x0601 -D_WIN32_IE=0x0501 -DWIN32_LEAN_AND_MEAN" dnl libtool insists upon adding -nostdlib and a list of objects/libs to link against. dnl That breaks our ability to build dll's with static libgcc/libstdc++/libssp. Override @@ -707,7 +712,7 @@ case $host in postdeps_CXX= dnl We require Windows 7 (NT 6.1) or later - AX_CHECK_LINK_FLAG([-Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1], [LDFLAGS="$LDFLAGS -Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1"], [], [$LDFLAG_WERROR]) ;; *darwin*) TARGET_OS=darwin @@ -745,20 +750,20 @@ case $host in if test "$use_upnp" != "no" && $BREW list --versions miniupnpc >/dev/null; then miniupnpc_prefix=$($BREW --prefix miniupnpc 2>/dev/null) if test "$suppress_external_warnings" != "no"; then - CPPFLAGS="$CPPFLAGS -isystem $miniupnpc_prefix/include" + CORE_CPPFLAGS="$CORE_CPPFLAGS -isystem $miniupnpc_prefix/include" else - CPPFLAGS="$CPPFLAGS -I$miniupnpc_prefix/include" + CORE_CPPFLAGS="$CORE_CPPFLAGS -I$miniupnpc_prefix/include" fi - LDFLAGS="$LDFLAGS -L$miniupnpc_prefix/lib" + CORE_LDFLAGS="$CORE_LDFLAGS -L$miniupnpc_prefix/lib" fi if test "$use_natpmp" != "no" && $BREW list --versions libnatpmp >/dev/null; then libnatpmp_prefix=$($BREW --prefix libnatpmp 2>/dev/null) if test "$suppress_external_warnings" != "no"; then - CPPFLAGS="$CPPFLAGS -isystem $libnatpmp_prefix/include" + CORE_CPPFLAGS="$CORE_CPPFLAGS -isystem $libnatpmp_prefix/include" else - CPPFLAGS="$CPPFLAGS -I$libnatpmp_prefix/include" + CORE_CPPFLAGS="$CORE_CPPFLAGS -I$libnatpmp_prefix/include" fi - LDFLAGS="$LDFLAGS -L$libnatpmp_prefix/lib" + CORE_LDFLAGS="$CORE_LDFLAGS -L$libnatpmp_prefix/lib" fi ;; esac @@ -784,8 +789,8 @@ case $host in esac fi - AX_CHECK_LINK_FLAG([-Wl,-headerpad_max_install_names], [LDFLAGS="$LDFLAGS -Wl,-headerpad_max_install_names"], [], [$LDFLAG_WERROR]) - CPPFLAGS="$CPPFLAGS -DMAC_OSX -DOBJC_OLD_DISPATCH_PROTOTYPES=0" + AX_CHECK_LINK_FLAG([-Wl,-headerpad_max_install_names], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-headerpad_max_install_names"], [], [$LDFLAG_WERROR]) + CORE_CPPFLAGS="$CORE_CPPFLAGS -DMAC_OSX -DOBJC_OLD_DISPATCH_PROTOTYPES=0" OBJCXXFLAGS="$CXXFLAGS" ;; *android*) @@ -848,11 +853,17 @@ if test "$use_lcov" = "yes"; then AC_SUBST(COV_TOOL_WRAPPER, "cov_tool_wrapper.sh") LCOV="$LCOV --gcov-tool $(pwd)/$COV_TOOL_WRAPPER" - AX_CHECK_LINK_FLAG([--coverage], [LDFLAGS="$LDFLAGS --coverage"], + AX_CHECK_LINK_FLAG([--coverage], [CORE_LDFLAGS="$CORE_LDFLAGS --coverage"], [AC_MSG_ERROR([lcov testing requested but --coverage linker flag does not work])]) - AX_CHECK_COMPILE_FLAG([--coverage],[CXXFLAGS="$CXXFLAGS --coverage"], + AX_CHECK_COMPILE_FLAG([--coverage],[CORE_CXXFLAGS="$CORE_CXXFLAGS --coverage"], [AC_MSG_ERROR([lcov testing requested but --coverage flag does not work])]) - CXXFLAGS="$CXXFLAGS -Og" + dnl If coverage is enabled, and the user hasn't overriden CXXFLAGS, clear + dnl them, to prevent autoconfs "-g -O2" being added. Otherwise we'd end up + dnl with "--coverage -Og -O0 -g -O2". + if test "$CXXFLAGS_overridden" = "no"; then + CXXFLAGS="" + fi + CORE_CXXFLAGS="$CORE_CXXFLAGS -Og -O0" fi if test "$use_lcov_branch" != "no"; then @@ -875,13 +886,13 @@ AC_FUNC_STRERROR_R if test "$ac_cv_sys_file_offset_bits" != "" && test "$ac_cv_sys_file_offset_bits" != "no" && test "$ac_cv_sys_file_offset_bits" != "unknown"; then - CPPFLAGS="$CPPFLAGS -D_FILE_OFFSET_BITS=$ac_cv_sys_file_offset_bits" + CORE_CPPFLAGS="$CORE_CPPFLAGS -D_FILE_OFFSET_BITS=$ac_cv_sys_file_offset_bits" fi if test "$ac_cv_sys_large_files" != "" && test "$ac_cv_sys_large_files" != "no" && test "$ac_cv_sys_large_files" != "unknown"; then - CPPFLAGS="$CPPFLAGS -D_LARGE_FILES=$ac_cv_sys_large_files" + CORE_CPPFLAGS="$CORE_CPPFLAGS -D_LARGE_FILES=$ac_cv_sys_large_files" fi AC_SEARCH_LIBS([clock_gettime],[rt]) @@ -965,8 +976,8 @@ dnl These flags are specific to ld64, and may cause issues with other linkers. dnl For example: GNU ld will interpret -dead_strip as -de and then try and use dnl "ad_strip" as the symbol for the entry point. if test "$TARGET_OS" = "darwin"; then - AX_CHECK_LINK_FLAG([-Wl,-dead_strip], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"], [], [$LDFLAG_WERROR]) - AX_CHECK_LINK_FLAG([-Wl,-dead_strip_dylibs], [LDFLAGS="$LDFLAGS -Wl,-dead_strip_dylibs"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-dead_strip], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-dead_strip"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-dead_strip_dylibs], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-dead_strip_dylibs"], [], [$LDFLAG_WERROR]) AX_CHECK_LINK_FLAG([-Wl,-bind_at_load], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-bind_at_load"], [], [$LDFLAG_WERROR]) fi @@ -976,7 +987,6 @@ AC_CHECK_DECLS([getifaddrs, freeifaddrs],[CHECK_SOCKET],, [#include <sys/types.h> #include <ifaddrs.h>] ) -AC_CHECK_DECLS([strnlen]) dnl These are used for daemonization in bitcoind AC_CHECK_DECLS([fork]) @@ -1266,7 +1276,7 @@ dnl Do not change "-I/usr/include" to "-isystem /usr/include" because that dnl is not necessary (/usr/include is already a system directory) and because dnl it would break GCC's #include_next. AC_DEFUN([SUPPRESS_WARNINGS], - [$(echo $1 |${SED} -E -e 's/(^| )-I/\1-isystem /g' -e 's;-isystem /usr/include([/ ]|$);-I/usr/include\1;g')]) + [[$(echo $1 |${SED} -E -e 's/(^| )-I/\1-isystem /g' -e 's;-isystem /usr/include/*( |$);-I/usr/include\1;g')]]) dnl enable-fuzz should disable all other targets if test "$enable_fuzz" = "yes"; then @@ -1295,7 +1305,7 @@ if test "$enable_fuzz" = "yes"; then AX_CHECK_LINK_FLAG( [-fsanitize=$use_sanitizers], [AC_MSG_RESULT([no])], - [AC_MSG_RESULT([yes]); CPPFLAGS="$CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION"], + [AC_MSG_RESULT([yes]); CORE_CPPFLAGS="$CORE_CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION"], [], [AC_LANG_PROGRAM([[ #include <cstdint> @@ -1319,7 +1329,7 @@ else QT_TEST_INCLUDES=SUPPRESS_WARNINGS($QT_TEST_INCLUDES) fi - CPPFLAGS="$CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION" + CORE_CPPFLAGS="$CORE_CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION" fi if test "$enable_wallet" != "no"; then @@ -1428,6 +1438,9 @@ if test "$use_boost" = "yes"; then AC_MSG_ERROR([only libbitcoinconsensus can be built without Boost]) fi + dnl we don't use multi_index serialization + BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_MULTI_INDEX_DISABLE_SERIALIZATION" + if test "$suppress_external_warnings" != "no"; then BOOST_CPPFLAGS=SUPPRESS_WARNINGS($BOOST_CPPFLAGS) fi @@ -1446,6 +1459,12 @@ if test "$use_external_signer" != "no"; then ;; *) AC_MSG_CHECKING([whether Boost.Process can be used]) + TEMP_CXXFLAGS="$CXXFLAGS" + dnl Boost 1.78 requires the following workaround. + dnl See: https://github.com/boostorg/process/issues/235 + CXXFLAGS="$CXXFLAGS -Wno-error=narrowing" + TEMP_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" TEMP_LDFLAGS="$LDFLAGS" dnl Boost 1.73 and older require the following workaround. LDFLAGS="$LDFLAGS $PTHREAD_CFLAGS" @@ -1453,6 +1472,8 @@ if test "$use_external_signer" != "no"; then [have_boost_process="yes"], [have_boost_process="no"]) LDFLAGS="$TEMP_LDFLAGS" + CPPFLAGS="$TEMP_CPPFLAGS" + CXXFLAGS="$TEMP_CXXFLAGS" AC_MSG_RESULT([$have_boost_process]) if test "$have_boost_process" = "yes"; then use_external_signer="yes" @@ -1500,7 +1521,7 @@ AM_CONDITIONAL([ENABLE_SYSCALL_SANDBOX], [test "$use_syscall_sandbox" != "no"]) dnl Check for reduced exports if test "$use_reduce_exports" = "yes"; then - AX_CHECK_COMPILE_FLAG([-fvisibility=hidden], [CXXFLAGS="$CXXFLAGS -fvisibility=hidden"], + AX_CHECK_COMPILE_FLAG([-fvisibility=hidden], [CORE_CXXFLAGS="$CORE_CXXFLAGS -fvisibility=hidden"], [AC_MSG_ERROR([Cannot set hidden symbol visibility. Use --disable-reduce-exports.])], [$CXXFLAG_WERROR]) AX_CHECK_LINK_FLAG([-Wl,--exclude-libs,ALL], [RELDFLAGS="-Wl,--exclude-libs,ALL"], [], [$LDFLAG_WERROR]) fi @@ -1861,6 +1882,9 @@ AC_SUBST(BITCOIN_MP_NODE_NAME) AC_SUBST(BITCOIN_MP_GUI_NAME) AC_SUBST(RELDFLAGS) +AC_SUBST(CORE_LDFLAGS) +AC_SUBST(CORE_CPPFLAGS) +AC_SUBST(CORE_CXXFLAGS) AC_SUBST(DEBUG_CPPFLAGS) AC_SUBST(WARN_CXXFLAGS) AC_SUBST(NOWARN_CXXFLAGS) @@ -1992,9 +2016,9 @@ echo " build os = $build_os" echo echo " CC = $CC" echo " CFLAGS = $PTHREAD_CFLAGS $CFLAGS" -echo " CPPFLAGS = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CPPFLAGS" +echo " CPPFLAGS = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CORE_CPPFLAGS $CPPFLAGS" echo " CXX = $CXX" -echo " CXXFLAGS = $LTO_CXXFLAGS $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CXXFLAGS" -echo " LDFLAGS = $LTO_LDFLAGS $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS" +echo " CXXFLAGS = $LTO_CXXFLAGS $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CORE_CXXFLAGS $CXXFLAGS" +echo " LDFLAGS = $LTO_LDFLAGS $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $CORE_LDFLAGS $LDFLAGS" echo " ARFLAGS = $ARFLAGS" echo diff --git a/contrib/builder-keys/keys.txt b/contrib/builder-keys/keys.txt index e8032f66ee..913e7d32f6 100644 --- a/contrib/builder-keys/keys.txt +++ b/contrib/builder-keys/keys.txt @@ -13,6 +13,7 @@ F20F56EF6A067F70E8A5C99FFF95FAA971697405 centaur (centaur) C060A6635913D98A3587D7DB1C2491FFEB0EF770 Cory Fields (cfields) BF6273FAEF7CC0BA1F562E50989F6B3048A116B5 Dev Random (devrandom) 6D3170C1DC2C6FD0AEEBCA6743811D1A26623924 Douglas Roark (droark) +948444FCE03B05BA5AB0591EC37B1C1D44C786EE Duncan Dean (dunxen) 1C6621605EC50319C463D56C7F81D87985D61612 Emanuele Cisbani (cisba) 9A1689B60D1B3CCE9262307A2F40A9BF167FBA47 Erik Mossberg (erkmos) D35176BE9264832E4ACA8986BF0792FBE95DC863 fivepiece (fivepiece) diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index e6a29b73b9..05c0af029e 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -12,10 +12,6 @@ from typing import List import lief #type:ignore -# temporary constant, to be replaced with lief.ELF.ARCH.RISCV -# https://github.com/lief-project/LIEF/pull/562 -LIEF_ELF_ARCH_RISCV = lief.ELF.ARCH(243) - def check_ELF_RELRO(binary) -> bool: ''' Check for read-only relocations. @@ -101,7 +97,6 @@ def check_ELF_separate_code(binary): for segment in binary.segments: if segment.type == lief.ELF.SEGMENT_TYPES.LOAD: for section in segment.sections: - assert(section.name not in flags_per_section) flags_per_section[section.name] = segment.flags # Spot-check ELF LOAD program header flags per section # If these sections exist, check them against the expected R/W/E flags @@ -222,7 +217,7 @@ CHECKS = { lief.ARCHITECTURES.ARM: BASE_ELF, lief.ARCHITECTURES.ARM64: BASE_ELF, lief.ARCHITECTURES.PPC: BASE_ELF, - LIEF_ELF_ARCH_RISCV: BASE_ELF, + lief.ARCHITECTURES.RISCV: BASE_ELF, }, lief.EXE_FORMATS.PE: { lief.ARCHITECTURES.X86: BASE_PE, @@ -250,12 +245,9 @@ if __name__ == '__main__': continue if arch == lief.ARCHITECTURES.NONE: - if binary.header.machine_type == LIEF_ELF_ARCH_RISCV: - arch = LIEF_ELF_ARCH_RISCV - else: - print(f'{filename}: unknown architecture') - retval = 1 - continue + print(f'{filename}: unknown architecture') + retval = 1 + continue failed: List[str] = [] for (name, func) in CHECKS[etype][arch]: diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 461132ae63..a419e392ee 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -15,10 +15,6 @@ from typing import List, Dict import lief #type:ignore -# temporary constant, to be replaced with lief.ELF.ARCH.RISCV -# https://github.com/lief-project/LIEF/pull/562 -LIEF_ELF_ARCH_RISCV = lief.ELF.ARCH(243) - # Debian 9 (Stretch) EOL: 2022. https://wiki.debian.org/DebianReleases#Production_Releases # # - g++ version 6.3.0 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=g%2B%2B) @@ -44,7 +40,7 @@ MAX_VERSIONS = { lief.ELF.ARCH.ARM: (2,18), lief.ELF.ARCH.AARCH64:(2,18), lief.ELF.ARCH.PPC64: (2,18), - LIEF_ELF_ARCH_RISCV: (2,27), + lief.ELF.ARCH.RISCV: (2,27), }, 'LIBATOMIC': (1,0), 'V': (0,5,0), # xkb (bitcoin-qt only) @@ -78,7 +74,7 @@ ELF_INTERPRETER_NAMES: Dict[lief.ELF.ARCH, Dict[lief.ENDIANNESS, str]] = { lief.ENDIANNESS.BIG: "/lib64/ld64.so.1", lief.ENDIANNESS.LITTLE: "/lib64/ld64.so.2", }, - LIEF_ELF_ARCH_RISCV: { + lief.ELF.ARCH.RISCV: { lief.ENDIANNESS.LITTLE: "/lib/ld-linux-riscv64-lp64d.so.1", }, } @@ -200,7 +196,7 @@ def check_exported_symbols(binary) -> bool: if not symbol.exported: continue name = symbol.name - if binary.header.machine_type == LIEF_ELF_ARCH_RISCV or name in IGNORE_EXPORTS: + if binary.header.machine_type == lief.ELF.ARCH.RISCV or name in IGNORE_EXPORTS: continue print(f'{binary.name}: export of symbol {name} not allowed!') ok = False diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py index e1a2ebc491..b4c112b266 100755 --- a/contrib/devtools/test-symbol-check.py +++ b/contrib/devtools/test-symbol-check.py @@ -187,7 +187,7 @@ class TestSymbolChecks(unittest.TestCase): executable = 'test3.exe' with open(source, 'w', encoding="utf8") as f: f.write(''' - #include <windows.h> + #include <combaseapi.h> int main() { diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 90289f9d40..af5607c710 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -75,7 +75,7 @@ crucial differences: 1. Since only Windows and macOS build outputs require codesigning, the `HOSTS` environment variable will have a sane default value of `x86_64-w64-mingw32 - x86_64-apple-darwin` instead of all the platforms. + x86_64-apple-darwin arm64-apple-darwin` instead of all the platforms. 2. The `guix-codesign` command ***requires*** a `DETACHED_SIGS_REPO` flag. * _**DETACHED_SIGS_REPO**_ diff --git a/contrib/guix/guix-codesign b/contrib/guix/guix-codesign index 94895d3fee..3279d431aa 100755 --- a/contrib/guix/guix-codesign +++ b/contrib/guix/guix-codesign @@ -152,10 +152,10 @@ outdir_for_host() { unsigned_tarball_for_host() { case "$1" in *mingw*) - echo "$(outdir_for_host "$1")/${DISTNAME}-win-unsigned.tar.gz" + echo "$(outdir_for_host "$1")/${DISTNAME}-win64-unsigned.tar.gz" ;; *darwin*) - echo "$(outdir_for_host "$1")/${DISTNAME}-osx-unsigned.tar.gz" + echo "$(outdir_for_host "$1")/${DISTNAME}-${1}-unsigned.tar.gz" ;; *) exit 1 diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index e06a469338..4eeb360603 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -167,7 +167,6 @@ case "$HOST" in *linux*) glibc_dynamic_linker=$( case "$HOST" in - i686-linux-gnu) echo /lib/ld-linux.so.2 ;; x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; @@ -204,19 +203,12 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \ ${SOURCES_PATH+SOURCES_PATH="$SOURCES_PATH"} \ ${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} \ ${SDK_PATH+SDK_PATH="$SDK_PATH"} \ - i686_linux_CC=i686-linux-gnu-gcc \ - i686_linux_CXX=i686-linux-gnu-g++ \ - i686_linux_AR=i686-linux-gnu-ar \ - i686_linux_RANLIB=i686-linux-gnu-ranlib \ - i686_linux_NM=i686-linux-gnu-nm \ - i686_linux_STRIP=i686-linux-gnu-strip \ x86_64_linux_CC=x86_64-linux-gnu-gcc \ x86_64_linux_CXX=x86_64-linux-gnu-g++ \ x86_64_linux_AR=x86_64-linux-gnu-ar \ x86_64_linux_RANLIB=x86_64-linux-gnu-ranlib \ x86_64_linux_NM=x86_64-linux-gnu-nm \ x86_64_linux_STRIP=x86_64-linux-gnu-strip \ - qt_config_opts_i686_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \ qt_config_opts_x86_64_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \ FORCE_USE_SYSTEM_CLANG=1 @@ -340,7 +332,7 @@ mkdir -p "$DISTSRC" mkdir -p "unsigned-app-${HOST}" cp --target-directory="unsigned-app-${HOST}" \ osx_volname \ - contrib/macdeploy/detached-sig-{apply,create}.sh \ + contrib/macdeploy/detached-sig-create.sh \ "${BASEPREFIX}/${HOST}"/native/bin/dmg mv --target-directory="unsigned-app-${HOST}" dist ( @@ -348,10 +340,10 @@ mkdir -p "$DISTSRC" find . -print0 \ | sort --zero-terminated \ | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz" && exit 1 ) + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" && exit 1 ) ) - make deploy ${V:+V=1} OSX_DMG="${OUTDIR}/${DISTNAME}-osx-unsigned.dmg" + make deploy ${V:+V=1} OSX_DMG="${OUTDIR}/${DISTNAME}-${HOST}-unsigned.dmg" ;; esac ( @@ -423,8 +415,8 @@ mkdir -p "$DISTSRC" find "${DISTNAME}" -print0 \ | sort --zero-terminated \ | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin/osx64}.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin/osx64}.tar.gz" && exit 1 ) + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) ;; esac ) # $DISTSRC/installed @@ -439,8 +431,8 @@ mkdir -p "$DISTSRC" find . -print0 \ | sort --zero-terminated \ | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" && exit 1 ) + | gzip -9n > "${OUTDIR}/${DISTNAME}-win64-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-win64-unsigned.tar.gz" && exit 1 ) ) ;; esac diff --git a/contrib/guix/libexec/codesign.sh b/contrib/guix/libexec/codesign.sh index a8867ca351..6ede95f42b 100755 --- a/contrib/guix/libexec/codesign.sh +++ b/contrib/guix/libexec/codesign.sh @@ -91,7 +91,7 @@ mkdir -p "$DISTSRC" -- -volume_date all_file_dates ="$SOURCE_DATE_EPOCH" # Compress uncompressed.dmg and output to OUTDIR - ./dmg dmg uncompressed.dmg "${OUTDIR}/${DISTNAME}-osx-signed.dmg" + ./dmg dmg uncompressed.dmg "${OUTDIR}/${DISTNAME}-${HOST}.dmg" ;; *) exit 1 diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index 086df48fbe..f24c120863 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -51,7 +51,7 @@ fi time-machine() { # shellcheck disable=SC2086 guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=ae03f401381e956c4c41b4cf495cbde964fa43d0 \ + --commit=34e9eae68c9583acce5abc4100add3d88932a5ae \ --cores="$JOBS" \ --keep-failed \ --fallback \ diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index a0afb7b4f6..fcec592c2c 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -162,9 +162,9 @@ desirable for building Bitcoin Core release binaries." (define (make-gcc-with-pthreads gcc) (package-with-extra-configure-variable gcc "--enable-threads" "posix")) -;; Required to support std::filesystem for mingw-w64 target. -(define (make-gcc-without-newlib gcc) - (package-with-extra-configure-variable gcc "--with-newlib" "no")) +(define (make-mingw-w64-cross-gcc-vmov-alignment cross-gcc) + (package-with-extra-patches cross-gcc + (search-our-patches "vmov-alignment.patch"))) (define (make-mingw-pthreads-cross-toolchain target) "Create a cross-compilation toolchain package for TARGET" @@ -172,7 +172,7 @@ desirable for building Bitcoin Core release binaries." (pthreads-xlibc mingw-w64-x86_64-winpthreads) (pthreads-xgcc (make-gcc-with-pthreads (cross-gcc target - #:xgcc (make-gcc-without-newlib (make-ssp-fixed-gcc base-gcc)) + #:xgcc (make-ssp-fixed-gcc (make-mingw-w64-cross-gcc-vmov-alignment base-gcc)) #:xbinutils xbinutils #:libc pthreads-xlibc)))) ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and @@ -201,7 +201,7 @@ chain for " target " development.")) (define-public lief (package (name "python-lief") - (version "0.11.5") + (version "0.12.0") (source (origin (method git-fetch) @@ -211,7 +211,7 @@ chain for " target " development.")) (file-name (git-file-name name version)) (sha256 (base32 - "0qahjfg1n0x76ps2mbyljvws1l3qhkqvmxqbahps4qgywl2hbdkj")))) + "026jchj56q25v6gc0754dj9cj5hz5zaza8ij93y5ga94w20kzm9q")))) (build-system python-build-system) (native-inputs `(("cmake" ,cmake))) @@ -485,7 +485,7 @@ and endian independent.") (license license:expat))) (define-public python-signapple - (let ((commit "0777ce58e61b0e6be753a5f524149d6d47905186")) + (let ((commit "8a945a2e7583be2665cf3a6a89d665b70ecd1ab6")) (package (name "python-signapple") (version (git-version "0.1" "1" commit)) @@ -498,7 +498,7 @@ and endian independent.") (file-name (git-file-name name commit)) (sha256 (base32 - "19axspyyfqbrfw2r53c17mi9bvm8zsb39mz8v9h7c173qkm3x5ym")))) + "0fr1hangvfyiwflca6jg5g8zvg3jc9qr7vd2c12ff89pznf38dlg")))) (build-system python-build-system) (propagated-inputs `(("python-asn1crypto" ,python-asn1crypto) @@ -506,8 +506,7 @@ and endian independent.") ("python-certvalidator" ,python-certvalidator) ("python-elfesteem" ,python-elfesteem) ("python-requests" ,python-requests) - ("python-macholib" ,python-macholib) - ("libcrypto" ,openssl))) + ("python-macholib" ,python-macholib))) ;; There are no tests, but attempting to run python setup.py test leads to ;; problems, just disable the test (arguments '(#:tests? #f)) diff --git a/contrib/guix/patches/vmov-alignment.patch b/contrib/guix/patches/vmov-alignment.patch new file mode 100644 index 0000000000..072f76eafd --- /dev/null +++ b/contrib/guix/patches/vmov-alignment.patch @@ -0,0 +1,267 @@ +Description: Use unaligned VMOV instructions +Author: Stephen Kitt <skitt@debian.org> +Bug-Debian: https://bugs.debian.org/939559 + +Based on a patch originally by Claude Heiland-Allen <claude@mathr.co.uk> + +--- a/gcc/config/i386/sse.md ++++ b/gcc/config/i386/sse.md +@@ -1058,17 +1058,11 @@ + { + if (FLOAT_MODE_P (GET_MODE_INNER (<MODE>mode))) + { +- if (misaligned_operand (operands[1], <MODE>mode)) +- return "vmovu<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; +- else +- return "vmova<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; ++ return "vmovu<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; + } + else + { +- if (misaligned_operand (operands[1], <MODE>mode)) +- return "vmovdqu<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; +- else +- return "vmovdqa<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; ++ return "vmovdqu<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; + } + } + [(set_attr "type" "ssemov") +@@ -1184,17 +1178,11 @@ + { + if (FLOAT_MODE_P (GET_MODE_INNER (<MODE>mode))) + { +- if (misaligned_operand (operands[0], <MODE>mode)) +- return "vmovu<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; +- else +- return "vmova<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; ++ return "vmovu<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; + } + else + { +- if (misaligned_operand (operands[0], <MODE>mode)) +- return "vmovdqu<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; +- else +- return "vmovdqa<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; ++ return "vmovdqu<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; + } + } + [(set_attr "type" "ssemov") +@@ -7806,7 +7794,7 @@ + "TARGET_SSE && !(MEM_P (operands[0]) && MEM_P (operands[1]))" + "@ + %vmovlps\t{%1, %0|%q0, %1} +- %vmovaps\t{%1, %0|%0, %1} ++ %vmovups\t{%1, %0|%0, %1} + %vmovlps\t{%1, %d0|%d0, %q1}" + [(set_attr "type" "ssemov") + (set_attr "prefix" "maybe_vex") +@@ -13997,29 +13985,15 @@ + switch (<MODE>mode) + { + case E_V8DFmode: +- if (misaligned_operand (operands[2], <ssequartermode>mode)) +- return "vmovupd\t{%2, %x0|%x0, %2}"; +- else +- return "vmovapd\t{%2, %x0|%x0, %2}"; ++ return "vmovupd\t{%2, %x0|%x0, %2}"; + case E_V16SFmode: +- if (misaligned_operand (operands[2], <ssequartermode>mode)) +- return "vmovups\t{%2, %x0|%x0, %2}"; +- else +- return "vmovaps\t{%2, %x0|%x0, %2}"; ++ return "vmovups\t{%2, %x0|%x0, %2}"; + case E_V8DImode: +- if (misaligned_operand (operands[2], <ssequartermode>mode)) +- return which_alternative == 2 ? "vmovdqu64\t{%2, %x0|%x0, %2}" ++ return which_alternative == 2 ? "vmovdqu64\t{%2, %x0|%x0, %2}" + : "vmovdqu\t{%2, %x0|%x0, %2}"; +- else +- return which_alternative == 2 ? "vmovdqa64\t{%2, %x0|%x0, %2}" +- : "vmovdqa\t{%2, %x0|%x0, %2}"; + case E_V16SImode: +- if (misaligned_operand (operands[2], <ssequartermode>mode)) +- return which_alternative == 2 ? "vmovdqu32\t{%2, %x0|%x0, %2}" ++ return which_alternative == 2 ? "vmovdqu32\t{%2, %x0|%x0, %2}" + : "vmovdqu\t{%2, %x0|%x0, %2}"; +- else +- return which_alternative == 2 ? "vmovdqa32\t{%2, %x0|%x0, %2}" +- : "vmovdqa\t{%2, %x0|%x0, %2}"; + default: + gcc_unreachable (); + } +@@ -21225,63 +21199,27 @@ + switch (get_attr_mode (insn)) + { + case MODE_V16SF: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- return "vmovups\t{%1, %t0|%t0, %1}"; +- else +- return "vmovaps\t{%1, %t0|%t0, %1}"; ++ return "vmovups\t{%1, %t0|%t0, %1}"; + case MODE_V8DF: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- return "vmovupd\t{%1, %t0|%t0, %1}"; +- else +- return "vmovapd\t{%1, %t0|%t0, %1}"; ++ return "vmovupd\t{%1, %t0|%t0, %1}"; + case MODE_V8SF: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- return "vmovups\t{%1, %x0|%x0, %1}"; +- else +- return "vmovaps\t{%1, %x0|%x0, %1}"; ++ return "vmovups\t{%1, %x0|%x0, %1}"; + case MODE_V4DF: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- return "vmovupd\t{%1, %x0|%x0, %1}"; +- else +- return "vmovapd\t{%1, %x0|%x0, %1}"; ++ return "vmovupd\t{%1, %x0|%x0, %1}"; + case MODE_XI: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- { +- if (which_alternative == 2) +- return "vmovdqu\t{%1, %t0|%t0, %1}"; +- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) +- return "vmovdqu64\t{%1, %t0|%t0, %1}"; +- else +- return "vmovdqu32\t{%1, %t0|%t0, %1}"; +- } ++ if (which_alternative == 2) ++ return "vmovdqu\t{%1, %t0|%t0, %1}"; ++ else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) ++ return "vmovdqu64\t{%1, %t0|%t0, %1}"; + else +- { +- if (which_alternative == 2) +- return "vmovdqa\t{%1, %t0|%t0, %1}"; +- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) +- return "vmovdqa64\t{%1, %t0|%t0, %1}"; +- else +- return "vmovdqa32\t{%1, %t0|%t0, %1}"; +- } ++ return "vmovdqu32\t{%1, %t0|%t0, %1}"; + case MODE_OI: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- { +- if (which_alternative == 2) +- return "vmovdqu\t{%1, %x0|%x0, %1}"; +- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) +- return "vmovdqu64\t{%1, %x0|%x0, %1}"; +- else +- return "vmovdqu32\t{%1, %x0|%x0, %1}"; +- } ++ if (which_alternative == 2) ++ return "vmovdqu\t{%1, %x0|%x0, %1}"; ++ else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) ++ return "vmovdqu64\t{%1, %x0|%x0, %1}"; + else +- { +- if (which_alternative == 2) +- return "vmovdqa\t{%1, %x0|%x0, %1}"; +- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) +- return "vmovdqa64\t{%1, %x0|%x0, %1}"; +- else +- return "vmovdqa32\t{%1, %x0|%x0, %1}"; +- } ++ return "vmovdqu32\t{%1, %x0|%x0, %1}"; + default: + gcc_unreachable (); + } +--- a/gcc/config/i386/i386.c ++++ b/gcc/config/i386/i386.c +@@ -4981,13 +4981,13 @@ + switch (type) + { + case opcode_int: +- opcode = misaligned_p ? "vmovdqu32" : "vmovdqa32"; ++ opcode = "vmovdqu32"; + break; + case opcode_float: +- opcode = misaligned_p ? "vmovups" : "vmovaps"; ++ opcode = "vmovups"; + break; + case opcode_double: +- opcode = misaligned_p ? "vmovupd" : "vmovapd"; ++ opcode = "vmovupd"; + break; + } + } +@@ -4996,16 +4996,16 @@ + switch (scalar_mode) + { + case E_SFmode: +- opcode = misaligned_p ? "%vmovups" : "%vmovaps"; ++ opcode = "%vmovups"; + break; + case E_DFmode: +- opcode = misaligned_p ? "%vmovupd" : "%vmovapd"; ++ opcode = "%vmovupd"; + break; + case E_TFmode: + if (evex_reg_p) +- opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64"; ++ opcode = "vmovdqu64"; + else +- opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa"; ++ opcode = "%vmovdqu"; + break; + default: + gcc_unreachable (); +@@ -5017,48 +5017,32 @@ + { + case E_QImode: + if (evex_reg_p) +- opcode = (misaligned_p +- ? (TARGET_AVX512BW +- ? "vmovdqu8" +- : "vmovdqu64") +- : "vmovdqa64"); ++ opcode = TARGET_AVX512BW ? "vmovdqu8" : "vmovdqu64"; + else +- opcode = (misaligned_p +- ? (TARGET_AVX512BW +- ? "vmovdqu8" +- : "%vmovdqu") +- : "%vmovdqa"); ++ opcode = TARGET_AVX512BW ? "vmovdqu8" : "%vmovdqu"; + break; + case E_HImode: + if (evex_reg_p) +- opcode = (misaligned_p +- ? (TARGET_AVX512BW +- ? "vmovdqu16" +- : "vmovdqu64") +- : "vmovdqa64"); ++ opcode = TARGET_AVX512BW ? "vmovdqu16" : "vmovdqu64"; + else +- opcode = (misaligned_p +- ? (TARGET_AVX512BW +- ? "vmovdqu16" +- : "%vmovdqu") +- : "%vmovdqa"); ++ opcode = TARGET_AVX512BW ? "vmovdqu16" : "%vmovdqu"; + break; + case E_SImode: + if (evex_reg_p) +- opcode = misaligned_p ? "vmovdqu32" : "vmovdqa32"; ++ opcode = "vmovdqu32"; + else +- opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa"; ++ opcode = "%vmovdqu"; + break; + case E_DImode: + case E_TImode: + case E_OImode: + if (evex_reg_p) +- opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64"; ++ opcode = "vmovdqu64"; + else +- opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa"; ++ opcode = "%vmovdqu"; + break; + case E_XImode: +- opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64"; ++ opcode = "vmovdqu64"; + break; + default: + gcc_unreachable (); diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index 441b5da764..7510204bb1 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -20,49 +20,9 @@ from collections import namedtuple settings = {} -def hex_switchEndian(s): - """ Switches the endianness of a hex string (in pairs of hex chars) """ - pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)] - return b''.join(pairList[::-1]).decode() - -def uint32(x): - return x & 0xffffffff - -def bytereverse(x): - return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) | - (((x) >> 8) & 0x0000ff00) | ((x) >> 24) )) - -def bufreverse(in_buf): - out_words = [] - for i in range(0, len(in_buf), 4): - word = struct.unpack('@I', in_buf[i:i+4])[0] - out_words.append(struct.pack('@I', bytereverse(word))) - return b''.join(out_words) - -def wordreverse(in_buf): - out_words = [] - for i in range(0, len(in_buf), 4): - out_words.append(in_buf[i:i+4]) - out_words.reverse() - return b''.join(out_words) - -def calc_hdr_hash(blk_hdr): - hash1 = hashlib.sha256() - hash1.update(blk_hdr) - hash1_o = hash1.digest() - - hash2 = hashlib.sha256() - hash2.update(hash1_o) - hash2_o = hash2.digest() - - return hash2_o - def calc_hash_str(blk_hdr): - hash = calc_hdr_hash(blk_hdr) - hash = bufreverse(hash) - hash = wordreverse(hash) - hash_str = hash.hex() - return hash_str + blk_hdr_hash = hashlib.sha256(hashlib.sha256(blk_hdr).digest()).digest() + return blk_hdr_hash[::-1].hex() def get_blk_dt(blk_hdr): members = struct.unpack("<I", blk_hdr[68:68+4]) @@ -78,7 +38,7 @@ def get_block_hashes(settings): for line in f: line = line.rstrip() if settings['rev_hash_bytes'] == 'true': - line = hex_switchEndian(line) + line = bytes.fromhex(line)[::-1].hex() blkindex.append(line) print("Read " + str(len(blkindex)) + " hashes") diff --git a/contrib/linearize/linearize-hashes.py b/contrib/linearize/linearize-hashes.py index fed6e665b8..0a316eb818 100755 --- a/contrib/linearize/linearize-hashes.py +++ b/contrib/linearize/linearize-hashes.py @@ -17,11 +17,6 @@ import os.path settings = {} -def hex_switchEndian(s): - """ Switches the endianness of a hex string (in pairs of hex chars) """ - pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)] - return b''.join(pairList[::-1]).decode() - class BitcoinRPC: def __init__(self, host, port, username, password): authpair = "%s:%s" % (username, password) @@ -85,7 +80,7 @@ def get_block_hashes(settings, max_blocks_per_call=10000): sys.exit(1) assert(resp_obj['id'] == x) # assume replies are in-sequence if settings['rev_hash_bytes'] == 'true': - resp_obj['result'] = hex_switchEndian(resp_obj['result']) + resp_obj['result'] = bytes.fromhex(resp_obj['result'])[::-1].hex() print(resp_obj['result']) height += num_blocks diff --git a/contrib/macdeploy/detached-sig-apply.sh b/contrib/macdeploy/detached-sig-apply.sh deleted file mode 100755 index c7296387eb..0000000000 --- a/contrib/macdeploy/detached-sig-apply.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -# Copyright (c) 2014-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C -set -e - -UNSIGNED="$1" -SIGNATURE="$2" -ROOTDIR=dist -OUTDIR=signed-app -SIGNAPPLE=signapple - -if [ -z "$UNSIGNED" ]; then - echo "usage: $0 <unsigned app> <signature>" - exit 1 -fi - -if [ -z "$SIGNATURE" ]; then - echo "usage: $0 <unsigned app> <signature>" - exit 1 -fi - -${SIGNAPPLE} apply "${UNSIGNED}" "${SIGNATURE}" -mv ${ROOTDIR} ${OUTDIR} -echo "Signed: ${OUTDIR}" diff --git a/contrib/signet/miner b/contrib/signet/miner index 012bd6cc31..b366b98e2d 100755 --- a/contrib/signet/miner +++ b/contrib/signet/miner @@ -8,7 +8,7 @@ import base64 import json import logging import math -import os.path +import os import re import struct import sys @@ -493,10 +493,11 @@ def do_generate(args): logging.debug("Mining block delta=%s start=%s mine=%s", seconds_to_hms(mine_time-bestheader["time"]), mine_time, is_mine) mined_blocks += 1 psbt = generate_psbt(tmpl, reward_spk, blocktime=mine_time) - psbt_signed = json.loads(args.bcli("-stdin", "walletprocesspsbt", input=psbt.encode('utf8'))) + input_stream = os.linesep.join([psbt, "true", "ALL"]).encode('utf8') + psbt_signed = json.loads(args.bcli("-stdin", "walletprocesspsbt", input=input_stream)) if not psbt_signed.get("complete",False): logging.debug("Generated PSBT: %s" % (psbt,)) - sys.stderr.write("PSBT signing failed") + sys.stderr.write("PSBT signing failed\n") return 1 block, signet_solution = do_decode_psbt(psbt_signed["psbt"]) block = finish_block(block, signet_solution, args.grind_cmd) diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index 99ca305fe7..6efe49254b 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -113,16 +113,6 @@ fun:GetCoin } { - Suppress boost warning - Memcheck:Leak - fun:_Znwm - ... - fun:_ZN5boost9unit_test9framework5state17execute_test_treeEmjPKNS2_23random_generator_helperE - fun:_ZN5boost9unit_test9framework3runEmb - fun:_ZN5boost9unit_test14unit_test_mainEPFbvEiPPc - fun:main -} -{ Suppress LogInstance still reachable memory warning Memcheck:Leak match-leak-kinds: reachable diff --git a/depends/hosts/openbsd.mk b/depends/hosts/openbsd.mk index dc8393e04c..5988f24bff 100644 --- a/depends/hosts/openbsd.mk +++ b/depends/hosts/openbsd.mk @@ -1,11 +1,11 @@ openbsd_CFLAGS=-pipe -openbsd_CFLAGS_CXXFLAGS=$(openbsd_CFLAGS) +openbsd_CXXFLAGS=$(openbsd_CFLAGS) -openbsd_CFLAGS_release_CFLAGS=-O2 -openbsd_CFLAGS_release_CXXFLAGS=$(openbsd_release_CFLAGS) +openbsd_release_CFLAGS=-O2 +openbsd_release_CXXFLAGS=$(openbsd_release_CFLAGS) -openbsd_CFLAGS_debug_CFLAGS=-O1 -openbsd_CFLAGS_debug_CXXFLAGS=$(openbsd_debug_CFLAGS) +openbsd_debug_CFLAGS=-O1 +openbsd_debug_CXXFLAGS=$(openbsd_debug_CFLAGS) ifeq (86,$(findstring 86,$(build_arch))) i686_openbsd_CC=clang -m32 diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index 748ed510c1..1efe6220d3 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -36,5 +36,6 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/*.la + rm lib/*.la && \ + rm include/ev*.h endef diff --git a/depends/packages/libmultiprocess.mk b/depends/packages/libmultiprocess.mk index 40ab3c68ea..864e33bc9a 100644 --- a/depends/packages/libmultiprocess.mk +++ b/depends/packages/libmultiprocess.mk @@ -6,7 +6,7 @@ $(package)_sha256_hash=$(native_$(package)_sha256_hash) $(package)_dependencies=native_$(package) capnp define $(package)_config_cmds - $($(package)_cmake) + $($(package)_cmake) . endef define $(package)_build_cmds diff --git a/depends/packages/native_libmultiprocess.mk b/depends/packages/native_libmultiprocess.mk index 14653ce9fb..6e600c5720 100644 --- a/depends/packages/native_libmultiprocess.mk +++ b/depends/packages/native_libmultiprocess.mk @@ -6,7 +6,7 @@ $(package)_sha256_hash=9f8b055c8bba755dc32fe799b67c20b91e7b13e67cadafbc54c0f1def $(package)_dependencies=native_capnp define $(package)_config_cmds - $($(package)_cmake) + $($(package)_cmake) . endef define $(package)_build_cmds diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 9cdfd21d2c..2bc3a81430 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -1,25 +1,32 @@ package=qt -$(package)_version=5.15.2 +$(package)_version=5.15.3 $(package)_download_path=https://download.qt.io/official_releases/qt/5.15/$($(package)_version)/submodules -$(package)_suffix=everywhere-src-$($(package)_version).tar.xz +$(package)_suffix=everywhere-opensource-src-$($(package)_version).tar.xz $(package)_file_name=qtbase-$($(package)_suffix) -$(package)_sha256_hash=909fad2591ee367993a75d7e2ea50ad4db332f05e1c38dd7a5a274e156a4e0f8 +$(package)_sha256_hash=26394ec9375d52c1592bd7b689b1619c6b8dbe9b6f91fdd5c355589787f3a0b6 $(package)_linux_dependencies=freetype fontconfig libxcb libxkbcommon libxcb_util libxcb_util_render libxcb_util_keysyms libxcb_util_image libxcb_util_wm $(package)_qt_libs=corelib network widgets gui plugins testlib $(package)_linguist_tools = lrelease lupdate lconvert -$(package)_patches = qt.pro qttools_src.pro -$(package)_patches += fix_qt_pkgconfig.patch mac-qmake.conf fix_no_printer.patch no-xlib.patch -$(package)_patches += dont_use_avx_android_x86_64.patch dont_hardcode_x86_64.patch fix_montery_include.patch -$(package)_patches += fix_android_jni_static.patch dont_hardcode_pwd.patch -$(package)_patches += qtbase-moc-ignore-gcc-macro.patch fix_limits_header.patch -$(package)_patches += fix_bigsur_style.patch use_android_ndk23.patch +$(package)_patches = qt.pro +$(package)_patches += qttools_src.pro +$(package)_patches += mac-qmake.conf +$(package)_patches += fix_qt_pkgconfig.patch +$(package)_patches += no-xlib.patch +$(package)_patches += dont_hardcode_x86_64.patch +$(package)_patches += fix_montery_include.patch +$(package)_patches += fix_android_jni_static.patch +$(package)_patches += dont_hardcode_pwd.patch +$(package)_patches += qtbase-moc-ignore-gcc-macro.patch +$(package)_patches += fix_limits_header.patch +$(package)_patches += use_android_ndk23.patch $(package)_patches += rcc_hardcode_timestamp.patch +$(package)_patches += duplicate_lcqpafonts.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) -$(package)_qttranslations_sha256_hash=d5788e86257b21d5323f1efd94376a213e091d1e5e03b45a95dd052b5f570db8 +$(package)_qttranslations_sha256_hash=5d7869f670a135ad0986e266813b9dd5bbae2b09577338f9cdf8904d4af52db0 $(package)_qttools_file_name=qttools-$($(package)_suffix) -$(package)_qttools_sha256_hash=c189d0ce1ff7c739db9a3ace52ac3e24cb8fd6dbf234e49f075249b38f43c1cc +$(package)_qttools_sha256_hash=463b2fe71a085e7ab4e39333ae360ab0ec857b966d7a08f752c427e5df55f90d $(package)_extra_sources = $($(package)_qttranslations_file_name) $(package)_extra_sources += $($(package)_qttools_file_name) @@ -232,17 +239,15 @@ define $(package)_preprocess_cmds cp $($(package)_patch_dir)/qttools_src.pro qttools/src/src.pro && \ patch -p1 -i $($(package)_patch_dir)/dont_hardcode_pwd.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_qt_pkgconfig.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_no_printer.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_android_jni_static.patch && \ patch -p1 -i $($(package)_patch_dir)/no-xlib.patch && \ - patch -p1 -i $($(package)_patch_dir)/dont_use_avx_android_x86_64.patch && \ patch -p1 -i $($(package)_patch_dir)/dont_hardcode_x86_64.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_limits_header.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_montery_include.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_bigsur_style.patch && \ patch -p1 -i $($(package)_patch_dir)/use_android_ndk23.patch && \ patch -p1 -i $($(package)_patch_dir)/rcc_hardcode_timestamp.patch && \ + patch -p1 -i $($(package)_patch_dir)/duplicate_lcqpafonts.patch && \ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ cp -f qtbase/mkspecs/macx-clang/qplatformdefs.h qtbase/mkspecs/macx-clang-linux/ &&\ cp -f $($(package)_patch_dir)/mac-qmake.conf qtbase/mkspecs/macx-clang-linux/qmake.conf && \ diff --git a/depends/patches/qt/dont_use_avx_android_x86_64.patch b/depends/patches/qt/dont_use_avx_android_x86_64.patch deleted file mode 100644 index b12ac8f4d0..0000000000 --- a/depends/patches/qt/dont_use_avx_android_x86_64.patch +++ /dev/null @@ -1,32 +0,0 @@ -Android: don't use avx and avx2 when building for Android x86_64 - -AVX and AVX2 are not supported on x86_64 ABI. -See: - - https://developer.android.com/ndk/guides/abis#86-64 - - https://bugreports.qt.io/browse/QTBUG-86785 - -Upstream commits: - - Qt 6.0: ff1a44be33f4bc05d502a2ca49171e0408992f61 - - Qt 5.15: 8b2cc0f6deb038a4c9d4f0d9b690c7726bd809a9 - - ---- old/qtbase/configure.json -+++ new/qtbase/configure.json -@@ -1098,7 +1098,7 @@ - }, - "avx": { - "label": "AVX", -- "condition": "features.sse4_2 && tests.avx", -+ "condition": "features.sse4_2 && tests.avx && (!config.android || !arch.x86_64)", - "output": [ - "privateConfig", - { "type": "define", "name": "QT_COMPILER_SUPPORTS_AVX", "value": 1 } -@@ -1114,7 +1114,7 @@ - }, - "avx2": { - "label": "AVX2", -- "condition": "features.avx && tests.avx2", -+ "condition": "features.avx && tests.avx2 && (!config.android || !arch.x86_64)", - "output": [ - "privateConfig", - "privateFeature", diff --git a/depends/patches/qt/duplicate_lcqpafonts.patch b/depends/patches/qt/duplicate_lcqpafonts.patch new file mode 100644 index 0000000000..c460b51dcf --- /dev/null +++ b/depends/patches/qt/duplicate_lcqpafonts.patch @@ -0,0 +1,104 @@ +QtGui: Fix duplication of logging category lcQpaFonts + +Move it to qplatformfontdatabase.h. + +Upstream commit: + - Qt 6.0: ab01885e48873fb2ad71841a3f1627fe4d9cd835 + +--- a/qtbase/src/gui/text/qplatformfontdatabase.cpp ++++ b/qtbase/src/gui/text/qplatformfontdatabase.cpp +@@ -52,6 +52,8 @@ + + QT_BEGIN_NAMESPACE + ++Q_LOGGING_CATEGORY(lcQpaFonts, "qt.qpa.fonts") ++ + void qt_registerFont(const QString &familyname, const QString &stylename, + const QString &foundryname, int weight, + QFont::Style style, int stretch, bool antialiased, + +--- a/qtbase/src/gui/text/qplatformfontdatabase.h ++++ b/qtbase/src/gui/text/qplatformfontdatabase.h +@@ -50,6 +50,7 @@ + // + + #include <QtGui/qtguiglobal.h> ++#include <QtCore/qloggingcategory.h> + #include <QtCore/QString> + #include <QtCore/QStringList> + #include <QtCore/QList> +@@ -62,6 +63,7 @@ + + QT_BEGIN_NAMESPACE + ++Q_DECLARE_LOGGING_CATEGORY(lcQpaFonts) + + class QWritingSystemsPrivate; + + +--- a/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm ++++ b/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm +@@ -86,8 +86,6 @@ + + QT_BEGIN_NAMESPACE + +-Q_LOGGING_CATEGORY(lcQpaFonts, "qt.qpa.fonts") +- + static float SYNTHETIC_ITALIC_SKEW = std::tan(14.f * std::acos(0.f) / 90.f); + + bool QCoreTextFontEngine::ct_getSfntTable(void *user_data, uint tag, uchar *buffer, uint *length) + +--- a/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext_p.h ++++ b/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext_p.h +@@ -64,8 +64,6 @@ + + QT_BEGIN_NAMESPACE + +-Q_DECLARE_LOGGING_CATEGORY(lcQpaFonts) +- + class QCoreTextFontEngine : public QFontEngine + { + Q_GADGET + +--- a/qtbase/src/platformsupport/fontdatabases/windows/qwindowsfontdatabase.cpp ++++ b/qtbase/src/platformsupport/fontdatabases/windows/qwindowsfontdatabase.cpp +@@ -68,8 +68,6 @@ + + QT_BEGIN_NAMESPACE + +-Q_LOGGING_CATEGORY(lcQpaFonts, "qt.qpa.fonts") +- + #ifndef QT_NO_DIRECTWRITE + // ### fixme: Consider direct linking of dwrite.dll once Windows Vista pre SP2 is dropped (QTBUG-49711) + + +--- a/qtbase/src/platformsupport/fontdatabases/windows/qwindowsfontdatabase_p.h ++++ b/qtbase/src/platformsupport/fontdatabases/windows/qwindowsfontdatabase_p.h +@@ -63,8 +63,6 @@ + + QT_BEGIN_NAMESPACE + +-Q_DECLARE_LOGGING_CATEGORY(lcQpaFonts) +- + class QWindowsFontEngineData + { + Q_DISABLE_COPY_MOVE(QWindowsFontEngineData) + +--- a/qtbase/src/platformsupport/themes/genericunix/qgenericunixthemes.cpp ++++ b/qtbase/src/platformsupport/themes/genericunix/qgenericunixthemes.cpp +@@ -40,6 +40,7 @@ + #include "qgenericunixthemes_p.h" + + #include "qpa/qplatformtheme_p.h" ++#include "qpa/qplatformfontdatabase.h" + + #include <QtGui/QPalette> + #include <QtGui/QFont> +@@ -76,7 +77,6 @@ + QT_BEGIN_NAMESPACE + + Q_DECLARE_LOGGING_CATEGORY(qLcTray) +-Q_LOGGING_CATEGORY(lcQpaFonts, "qt.qpa.fonts") + + ResourceHelper::ResourceHelper() + { diff --git a/depends/patches/qt/fix_android_jni_static.patch b/depends/patches/qt/fix_android_jni_static.patch index bb64661761..22a4d5ab0e 100644 --- a/depends/patches/qt/fix_android_jni_static.patch +++ b/depends/patches/qt/fix_android_jni_static.patch @@ -1,6 +1,6 @@ --- old/qtbase/src/plugins/platforms/android/androidjnimain.cpp +++ new/qtbase/src/plugins/platforms/android/androidjnimain.cpp -@@ -914,6 +914,14 @@ Q_DECL_EXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void */*reserved*/) +@@ -934,6 +934,14 @@ Q_DECL_EXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void */*reserved*/) __android_log_print(ANDROID_LOG_FATAL, "Qt", "registerNatives failed"); return -1; } diff --git a/depends/patches/qt/fix_bigsur_style.patch b/depends/patches/qt/fix_bigsur_style.patch deleted file mode 100644 index 0f6c848bb0..0000000000 --- a/depends/patches/qt/fix_bigsur_style.patch +++ /dev/null @@ -1,90 +0,0 @@ -Fix rendering in macOS BigSur - -See: https://bugreports.qt.io/browse/QTBUG-86513. - -Upstream commits (combined in this patch): - - Qt 6.0: 40fb97e97f550b8afd13ecc3a038d9d0c2d82bbb - - Qt 6.0: 3857f104cac127f62e64e55a20613f0ac2e6b843 - - Qt 6.1: abee4cdd5925a8513f51784754fca8fa35031732 - ---- old/qtbase/src/plugins/styles/mac/qmacstyle_mac.mm -+++ new/qtbase/src/plugins/styles/mac/qmacstyle_mac.mm -@@ -3870,6 +3870,7 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter - const auto cs = d->effectiveAquaSizeConstrain(opt, w); - // Extra hacks to get the proper pressed appreance when not selected or selected and inactive - const bool needsInactiveHack = (!isActive && isSelected); -+ const bool isBigSurOrAbove = QOperatingSystemVersion::current() >= QOperatingSystemVersion::MacOSBigSur; - const auto ct = !needsInactiveHack && (isSelected || tp == QStyleOptionTab::OnlyOneTab) ? - QMacStylePrivate::Button_PushButton : - QMacStylePrivate::Button_PopupButton; -@@ -3878,6 +3879,12 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter - auto *pb = static_cast<NSButton *>(d->cocoaControl(cw)); - - auto vOffset = isPopupButton ? 1 : 2; -+ if (isBigSurOrAbove) { -+ // Make it 1, otherwise, offset is very visible compared -+ // to selected tab (which is not a popup button). -+ vOffset = 1; -+ } -+ - if (tabDirection == QMacStylePrivate::East) - vOffset -= 1; - const auto outerAdjust = isPopupButton ? 1 : 4; -@@ -3894,9 +3901,22 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter - frameRect = frameRect.adjusted(-innerAdjust, 0, outerAdjust, 0); - else - frameRect = frameRect.adjusted(-outerAdjust, 0, innerAdjust, 0); -+ -+ if (isSelected && isBigSurOrAbove) { -+ // 1 pixed of 'roundness' is still visible on the right -+ // (the left is OK, it's rounded). -+ frameRect = frameRect.adjusted(0, 0, 1, 0); -+ } -+ - break; - case QStyleOptionTab::Middle: - frameRect = frameRect.adjusted(-innerAdjust, 0, innerAdjust, 0); -+ -+ if (isSelected && isBigSurOrAbove) { -+ // 1 pixel of 'roundness' is still visible on both -+ // sides - left and right. -+ frameRect = frameRect.adjusted(-1, 0, 1, 0); -+ } - break; - case QStyleOptionTab::End: - // Pressed state hack: tweak adjustments in preparation for flip below -@@ -3904,6 +3924,11 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter - frameRect = frameRect.adjusted(-innerAdjust, 0, outerAdjust, 0); - else - frameRect = frameRect.adjusted(-outerAdjust, 0, innerAdjust, 0); -+ -+ if (isSelected && isBigSurOrAbove) { -+ // 1 pixel of 'roundness' is still visible on the left. -+ frameRect = frameRect.adjusted(-1, 0, 0, 0); -+ } - break; - case QStyleOptionTab::OnlyOneTab: - frameRect = frameRect.adjusted(-outerAdjust, 0, outerAdjust, 0); -@@ -3951,7 +3976,10 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter - NSPopUpArrowPosition oldPosition = NSPopUpArrowAtCenter; - NSPopUpButtonCell *pbCell = nil; - auto rAdjusted = r; -- if (isPopupButton && tp == QStyleOptionTab::OnlyOneTab) { -+ if (isPopupButton && (tp == QStyleOptionTab::OnlyOneTab || isBigSurOrAbove)) { -+ // Note: starting from macOS BigSur NSPopupButton has this -+ // arrow 'button' in a different place and it became -+ // quite visible 'in between' inactive tabs. - pbCell = static_cast<NSPopUpButtonCell *>(pb.cell); - oldPosition = pbCell.arrowPosition; - pbCell.arrowPosition = NSPopUpNoArrow; -@@ -3959,6 +3987,10 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter - // NSPopUpButton in this state is smaller. - rAdjusted.origin.x -= 3; - rAdjusted.size.width += 6; -+ if (isBigSurOrAbove) { -+ if (tp == QStyleOptionTab::End) -+ rAdjusted.origin.x -= 2; -+ } - } - } - diff --git a/depends/patches/qt/fix_limits_header.patch b/depends/patches/qt/fix_limits_header.patch index cb5a8cd1b5..258128c0ca 100644 --- a/depends/patches/qt/fix_limits_header.patch +++ b/depends/patches/qt/fix_limits_header.patch @@ -1,46 +1,9 @@ Fix compiling with GCC 11 -See: https://bugreports.qt.io/browse/QTBUG-90395. +Upstream: + - bug report: https://bugreports.qt.io/browse/QTBUG-89977 + - fix in Qt 6.1: 813a928c7c3cf98670b6043149880ed5c955efb9 -Upstream commits: - - Qt 5.15 -- unavailable as open source - - Qt 6.0: b2af6332ea37e45ab230a7a5d2d278f86d961b83 - - Qt 6.1: 9c56d4da2ff631a8c1c30475bd792f6c86bda53c - ---- old/qtbase/src/corelib/global/qendian.h -+++ new/qtbase/src/corelib/global/qendian.h -@@ -44,6 +44,8 @@ - #include <QtCore/qfloat16.h> - #include <QtCore/qglobal.h> - -+#include <limits> -+ - // include stdlib.h and hope that it defines __GLIBC__ for glibc-based systems - #include <stdlib.h> - #include <string.h> - ---- old/qtbase/src/corelib/global/qfloat16.h -+++ new/qtbase/src/corelib/global/qfloat16.h -@@ -43,6 +43,7 @@ - - #include <QtCore/qglobal.h> - #include <QtCore/qmetatype.h> -+#include <limits> - #include <string.h> - - #if defined(QT_COMPILER_SUPPORTS_F16C) && defined(__AVX2__) && !defined(__F16C__) - ---- old/qtbase/src/tools/moc/generator.cpp -+++ new/qtbase/src/tools/moc/generator.cpp -@@ -40,6 +40,8 @@ - #include <QtCore/qplugin.h> - #include <QtCore/qstringview.h> - -+#include <limits> -+ - #include <math.h> - #include <stdio.h> - --- old/qtbase/src/corelib/text/qbytearraymatcher.h +++ new/qtbase/src/corelib/text/qbytearraymatcher.h @@ -42,6 +42,8 @@ @@ -52,6 +15,12 @@ Upstream commits: QT_BEGIN_NAMESPACE + +Upstream fix and backports: + - Qt 6.1: 3eab20ad382569cb2c9e6ccec2322c3d08c0f716 + - Qt 6.2: 380294a5971da85010a708dc23b0edec192cbf27 + - Qt 6.3: 2b2b3155d9f6ba1e4f859741468fbc47db09292b + --- old/qtbase/src/corelib/tools/qoffsetstringarray_p.h +++ new/qtbase/src/corelib/tools/qoffsetstringarray_p.h @@ -55,6 +55,7 @@ diff --git a/depends/patches/qt/fix_no_printer.patch b/depends/patches/qt/fix_no_printer.patch deleted file mode 100644 index 1372356138..0000000000 --- a/depends/patches/qt/fix_no_printer.patch +++ /dev/null @@ -1,19 +0,0 @@ ---- x/qtbase/src/plugins/platforms/cocoa/qprintengine_mac_p.h -+++ y/qtbase/src/plugins/platforms/cocoa/qprintengine_mac_p.h -@@ -52,6 +52,7 @@ - // - - #include <QtCore/qglobal.h> -+#include <qpa/qplatformprintdevice.h> - - #ifndef QT_NO_PRINTER - ---- x/qtbase/src/plugins/plugins.pro -+++ y/qtbase/src/plugins/plugins.pro -@@ -9,6 +9,3 @@ qtHaveModule(gui) { - !android:qtConfig(library): SUBDIRS *= generic - } - qtHaveModule(widgets): SUBDIRS += styles -- --!winrt:qtHaveModule(printsupport): \ -- SUBDIRS += printsupport diff --git a/depends/patches/qt/mac-qmake.conf b/depends/patches/qt/mac-qmake.conf index e4bfaa1463..543407f853 100644 --- a/depends/patches/qt/mac-qmake.conf +++ b/depends/patches/qt/mac-qmake.conf @@ -19,6 +19,4 @@ QMAKE_MAC_SDK.macosx.PlatformPath = /phony !host_build: QMAKE_LFLAGS += -target $${MAC_TARGET} QMAKE_AR = $${CROSS_COMPILE}ar cq QMAKE_RANLIB=$${CROSS_COMPILE}ranlib -QMAKE_LIBTOOL=$${CROSS_COMPILE}libtool -QMAKE_INSTALL_NAME_TOOL=$${CROSS_COMPILE}install_name_tool load(qt_config) diff --git a/depends/patches/qt/use_android_ndk23.patch b/depends/patches/qt/use_android_ndk23.patch index 85b8690218..f22367d527 100644 --- a/depends/patches/qt/use_android_ndk23.patch +++ b/depends/patches/qt/use_android_ndk23.patch @@ -2,7 +2,7 @@ Use Android NDK r23 LTS --- old/qtbase/mkspecs/features/android/default_pre.prf +++ new/qtbase/mkspecs/features/android/default_pre.prf -@@ -73,7 +73,7 @@ else: equals(QT_ARCH, x86_64): CROSS_COMPILE = $$NDK_LLVM_PATH/bin/x86_64-linux- +@@ -76,7 +76,7 @@ else: equals(QT_ARCH, x86_64): CROSS_COMPILE = $$NDK_LLVM_PATH/bin/x86_64-linux- else: equals(QT_ARCH, arm64-v8a): CROSS_COMPILE = $$NDK_LLVM_PATH/bin/aarch64-linux-android- else: CROSS_COMPILE = $$NDK_LLVM_PATH/bin/arm-linux-androideabi- diff --git a/doc/README.md b/doc/README.md index c200ac3753..33f71f4807 100644 --- a/doc/README.md +++ b/doc/README.md @@ -73,6 +73,7 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th - [Assets Attribution](assets-attribution.md) - [Assumeutxo design](assumeutxo.md) - [bitcoin.conf Configuration File](bitcoin-conf.md) +- [CJDNS Support](cjdns.md) - [Files](files.md) - [Fuzz-testing](fuzzing.md) - [I2P Support](i2p.md) diff --git a/doc/build-freebsd.md b/doc/build-freebsd.md index da2ab61c2a..a8e643a2ab 100644 --- a/doc/build-freebsd.md +++ b/doc/build-freebsd.md @@ -1,48 +1,21 @@ # FreeBSD Build Guide -**Updated for FreeBSD [12.2](https://www.freebsd.org/releases/12.2R/announce.html)** +**Updated for FreeBSD [12.3](https://www.freebsd.org/releases/12.3R/announce/)** This guide describes how to build bitcoind, command-line utilities, and GUI on FreeBSD. -## Dependencies - -The following dependencies are **required**: - - Library | Purpose | Description - ----------------------------------------------------------------------|------------|---------------------- - [autoconf](https://svnweb.freebsd.org/ports/head/devel/autoconf/) | Build | Automatically configure software source code - [automake](https://svnweb.freebsd.org/ports/head/devel/automake/) | Build | Generate makefile (requires autoconf) - [libtool](https://svnweb.freebsd.org/ports/head/devel/libtool/) | Build | Shared library support - [pkgconf](https://svnweb.freebsd.org/ports/head/devel/pkgconf/) | Build | Configure compiler and linker flags - [git](https://svnweb.freebsd.org/ports/head/devel/git/) | Clone | Version control system - [gmake](https://svnweb.freebsd.org/ports/head/devel/gmake/) | Compile | Generate executables - [boost-libs](https://svnweb.freebsd.org/ports/head/devel/boost-libs/) | Utility | Library for threading, data structures, etc - [libevent](https://svnweb.freebsd.org/ports/head/devel/libevent/) | Networking | OS independent asynchronous networking - - -The following dependencies are **optional**: - - Library | Purpose | Description - ---------------------------------------------------------------------------|------------------|---------------------- - [db5](https://svnweb.freebsd.org/ports/head/databases/db5/) | Berkeley DB | Wallet storage (only needed when wallet enabled) - [qt5](https://svnweb.freebsd.org/ports/head/devel/qt5/) | GUI | GUI toolkit (only needed when GUI enabled) - [libqrencode](https://svnweb.freebsd.org/ports/head/graphics/libqrencode/) | QR codes in GUI | Generating QR codes (only needed when GUI enabled) - [libzmq4](https://svnweb.freebsd.org/ports/head/net/libzmq4/) | ZMQ notification | Allows generating ZMQ notifications (requires ZMQ version >= 4.0.0) - [sqlite3](https://svnweb.freebsd.org/ports/head/databases/sqlite3/) | SQLite DB | Wallet storage (only needed when wallet enabled) - [python3](https://svnweb.freebsd.org/ports/head/lang/python3/) | Testing | Python Interpreter (only needed when running the test suite) - - See [dependencies.md](dependencies.md) for a complete overview. - ## Preparation ### 1. Install Required Dependencies -Install the required dependencies the usual way you [install software on FreeBSD](https://www.freebsd.org/doc/en/books/handbook/ports.html) - either with `pkg` or via the Ports collection. The example commands below use `pkg` which is usually run as `root` or via `sudo`. If you want to use `sudo`, and you haven't set it up: [use this guide](http://www.freebsdwiki.net/index.php/Sudo%2C_configuring) to setup `sudo` access on FreeBSD. +Run the following as root to install the base dependencies for building. ```bash pkg install autoconf automake boost-libs git gmake libevent libtool pkgconf ``` +See [dependencies.md](dependencies.md) for a complete overview. + ### 2. Clone Bitcoin Repo Now that `git` and all the required dependencies are installed, let's clone the Bitcoin Core repository to a directory. All build scripts and commands will run from this directory. ``` bash @@ -52,21 +25,23 @@ git clone https://github.com/bitcoin/bitcoin.git ### 3. Install Optional Dependencies #### Wallet Dependencies -It is not necessary to build wallet functionality to run bitcoind or the GUI. To enable legacy wallets, you must install `db5`. To enable [descriptor wallets](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md), `sqlite3` is required. Skip `db5` if you intend to *exclusively* use descriptor wallets - -###### Legacy Wallet Support -`db5` is required to enable support for legacy wallets. Skip if you don't intend to use legacy wallets - -```bash -pkg install db5 -``` +It is not necessary to build wallet functionality to run either `bitcoind` or `bitcoin-qt`. ###### Descriptor Wallet Support -`sqlite3` is required to enable support for descriptor wallets. Skip if you don't intend to use descriptor wallets. +`sqlite3` is required to support [descriptor wallets](descriptors.md). +Skip if you don't intend to use descriptor wallets. ``` bash pkg install sqlite3 ``` + +###### Legacy Wallet Support +`db5` is only required to support legacy wallets. +Skip if you don't intend to use legacy wallets. + +```bash +pkg install db5 +``` --- #### GUI Dependencies @@ -84,6 +59,14 @@ pkg install libqrencode ``` --- +#### Notifications +###### ZeroMQ + +Bitcoin Core can provide notifications via ZeroMQ. If the package is installed, support will be compiled in. +```bash +pkg install libzmq4 +``` + #### Test Suite Dependencies There is an included test suite that is useful for testing code changes when developing. To run the test suite (recommended), you will need to have Python 3 installed: @@ -98,8 +81,17 @@ pkg install python3 ### 1. Configuration There are many ways to configure Bitcoin Core, here are a few common examples: -##### Wallet (BDB + SQlite) Support, No GUI: -This explicitly enables legacy wallet support and disables the GUI. If `sqlite3` is installed, then descriptor wallet support will be built. + +##### Descriptor Wallet and GUI: +This explicitly enables the GUI and disables legacy wallet support, assuming `sqlite` and `qt` are installed. +```bash +./autogen.sh +./configure --without-bdb --with-gui=yes MAKE=gmake +``` + +##### Descriptor & Legacy Wallet. No GUI: +This enables support for both wallet types and disables the GUI, assuming +`sqlite3` and `db5` are both installed. ```bash ./autogen.sh ./configure --with-gui=no --with-incompatible-bdb \ @@ -108,12 +100,6 @@ This explicitly enables legacy wallet support and disables the GUI. If `sqlite3` MAKE=gmake ``` -##### Wallet (only SQlite) and GUI Support: -This explicitly enables the GUI and disables legacy wallet support. If `qt5` is not installed, this will throw an error. If `sqlite3` is installed then descriptor wallet functionality will be built. If `sqlite3` is not installed, then wallet functionality will be disabled. -```bash -./autogen.sh -./configure --without-bdb --with-gui=yes MAKE=gmake -``` ##### No Wallet or GUI ``` bash ./autogen.sh diff --git a/doc/build-netbsd.md b/doc/build-netbsd.md index edabd71611..ba9a80f83b 100644 --- a/doc/build-netbsd.md +++ b/doc/build-netbsd.md @@ -27,15 +27,33 @@ git clone https://github.com/bitcoin/bitcoin.git See [dependencies.md](dependencies.md) for a complete overview. -### Building BerkeleyDB +### Building Bitcoin Core + +**Important**: Use `gmake` (the non-GNU `make` will exit with an error). + +#### With descriptor wallet: + +The descriptor wallet uses `sqlite3`. You can install it using: +```bash +pkgin install sqlite3 +``` + +```bash +./autogen.sh +./configure --with-gui=no --without-bdb \ + CPPFLAGS="-I/usr/pkg/include" \ + LDFLAGS="-L/usr/pkg/lib" \ + BOOST_CPPFLAGS="-I/usr/pkg/include" \ + MAKE=gmake +``` -BerkeleyDB is only necessary for the wallet functionality. To skip this, pass -`--disable-wallet` to `./configure` and skip to the next section. +#### With legacy wallet: + +BerkeleyDB is use for legacy wallet functionality. It is recommended to use Berkeley DB 4.8. You cannot use the BerkeleyDB library -from ports, for the same reason as boost above (g++/libstd++ incompatibility). -If you have to build it yourself, you can use [the installation script included -in contrib/](/contrib/install_db4.sh) like so: +from ports. +You can use [the installation script included in contrib/](/contrib/install_db4.sh) like so: ```bash ./contrib/install_db4.sh `pwd` @@ -47,30 +65,23 @@ from the root of the repository. Then set `BDB_PREFIX` for the next section: export BDB_PREFIX="$PWD/db4" ``` -### Building Bitcoin Core - -**Important**: Use `gmake` (the non-GNU `make` will exit with an error). - -With wallet: ```bash ./autogen.sh ./configure --with-gui=no CPPFLAGS="-I/usr/pkg/include" \ LDFLAGS="-L/usr/pkg/lib" \ BOOST_CPPFLAGS="-I/usr/pkg/include" \ - BOOST_LDFLAGS="-L/usr/pkg/lib" \ BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ BDB_CFLAGS="-I${BDB_PREFIX}/include" \ MAKE=gmake ``` -Without wallet: +#### Without wallet: ```bash ./autogen.sh ./configure --with-gui=no --disable-wallet \ CPPFLAGS="-I/usr/pkg/include" \ LDFLAGS="-L/usr/pkg/lib" \ BOOST_CPPFLAGS="-I/usr/pkg/include" \ - BOOST_LDFLAGS="-L/usr/pkg/lib" \ MAKE=gmake ``` diff --git a/doc/build-osx.md b/doc/build-osx.md index 16dc224aed..fdf0a9d414 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -4,42 +4,6 @@ This guide describes how to build bitcoind, command-line utilities, and GUI on macOS -**Note:** The following is for Intel Macs only! - -## Dependencies - -The following dependencies are **required**: - -Library | Purpose | Description ------------------------------------------------------------|------------|---------------------- -[automake](https://formulae.brew.sh/formula/automake) | Build | Generate makefile -[libtool](https://formulae.brew.sh/formula/libtool) | Build | Shared library support -[pkg-config](https://formulae.brew.sh/formula/pkg-config) | Build | Configure compiler and linker flags -[boost](https://formulae.brew.sh/formula/boost) | Utility | Library for threading, data structures, etc -[libevent](https://formulae.brew.sh/formula/libevent) | Networking | OS independent asynchronous networking - -The following dependencies are **optional**: - -Library | Purpose | Description ---------------------------------------------------------------- |------------------|---------------------- -[berkeley-db@4](https://formulae.brew.sh/formula/berkeley-db@4) | Berkeley DB | Wallet storage (only needed when wallet enabled) -[qt@5](https://formulae.brew.sh/formula/qt@5) | GUI | GUI toolkit (only needed when GUI enabled) -[qrencode](https://formulae.brew.sh/formula/qrencode) | QR codes in GUI | Generating QR codes (only needed when GUI enabled) -[zeromq](https://formulae.brew.sh/formula/zeromq) | ZMQ notification | Allows generating ZMQ notifications (requires ZMQ version >= 4.0.0) -[sqlite](https://formulae.brew.sh/formula/sqlite) | SQLite DB | Wallet storage (only needed when wallet enabled) -[miniupnpc](https://formulae.brew.sh/formula/miniupnpc) | UPnP Support | Firewall-jumping support (needed for port mapping support) -[libnatpmp](https://formulae.brew.sh/formula/libnatpmp) | NAT-PMP Support | Firewall-jumping support (needed for port mapping support) -[python3](https://formulae.brew.sh/formula/python@3.9) | Testing | Python Interpreter (only needed when running the test suite) - -The following dependencies are **optional** packages required for deploying: - -Library | Purpose | Description -----------------------------------------------------|------------------|---------------------- -[ds_store](https://pypi.org/project/ds-store/) | Deploy Dependency| Examine and modify .DS_Store files -[mac_alias](https://pypi.org/project/mac-alias/) | Deploy Dependency| Generate/Read binary alias and bookmark records - -See [dependencies.md](dependencies.md) for a complete overview. - ## Preparation The commands in this guide should be executed in a Terminal application. @@ -78,6 +42,9 @@ Note: If you run into issues while installing Homebrew or pulling packages, refe The first step is to download the required dependencies. These dependencies represent the packages required to get a barebones installation up and running. + +See [dependencies.md](dependencies.md) for a complete overview. + To install, run the following from your terminal: ``` bash @@ -99,29 +66,21 @@ git clone https://github.com/bitcoin/bitcoin.git #### Wallet Dependencies It is not necessary to build wallet functionality to run `bitcoind` or `bitcoin-qt`. -To enable legacy wallets, you must install `berkeley-db@4`. -To enable [descriptor wallets](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md), `sqlite` is required. -Skip `berkeley-db@4` if you intend to *exclusively* use descriptor wallets. - -###### Legacy Wallet Support -`berkeley-db@4` is required to enable support for legacy wallets. -Skip if you don't intend to use legacy wallets. +###### Descriptor Wallet Support -``` bash -brew install berkeley-db@4 -``` +`sqlite` is required to support for descriptor wallets. -###### Descriptor Wallet Support +macOS ships with a useable `sqlite` package, meaning you don't need to +install anything. -Note: Apple has included a useable `sqlite` package since macOS 10.14. -You may not need to install this package. +###### Legacy Wallet Support -`sqlite` is required to enable support for descriptor wallets. -Skip if you don't intend to use descriptor wallets. +`berkeley-db@4` is only required to support for legacy wallets. +Skip if you don't intend to use legacy wallets. ``` bash -brew install sqlite +brew install berkeley-db@4 ``` --- diff --git a/doc/build-unix.md b/doc/build-unix.md index 15fe63d047..f02729ee32 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -26,30 +26,7 @@ make install # optional This will build bitcoin-qt as well, if the dependencies are met. -Dependencies ---------------------- - -These dependencies are required: - - Library | Purpose | Description - ------------|------------------|---------------------- - libboost | Utility | Library for threading, data structures, etc - libevent | Networking | OS independent asynchronous networking - -Optional dependencies: - - Library | Purpose | Description - ------------|------------------|---------------------- - miniupnpc | UPnP Support | Firewall-jumping support - libnatpmp | NAT-PMP Support | Firewall-jumping support - libdb4.8 | Berkeley DB | Wallet storage (only needed when legacy wallet enabled) - qt | GUI | GUI toolkit (only needed when GUI enabled) - libqrencode | QR codes in GUI | QR code generation (only needed when GUI enabled) - libzmq3 | ZMQ notification | ZMQ notifications (requires ZMQ version >= 4.0.0) - sqlite3 | SQLite DB | Wallet storage (only needed when descriptor wallet enabled) - systemtap | Tracing (USDT) | Statically defined tracepoints - -For the versions used, see [dependencies.md](dependencies.md) +See [dependencies.md](dependencies.md) for a complete overview. Memory Requirements -------------------- @@ -232,7 +209,7 @@ from the root of the repository. Otherwise, you can build Bitcoin Core from self-compiled [depends](/depends/README.md). -**Note**: You only need Berkeley DB if the wallet is enabled (see [*Disable-wallet mode*](#disable-wallet-mode)). +**Note**: You only need Berkeley DB if the legacy wallet is enabled (see [*Disable-wallet mode*](#disable-wallet-mode)). Security -------- @@ -282,12 +259,12 @@ Hardening enables the following features: Disable-wallet mode -------------------- -When the intention is to run only a P2P node without a wallet, Bitcoin Core may be compiled in -disable-wallet mode with: +When the intention is to only run a P2P node, without a wallet, Bitcoin Core can +be compiled in disable-wallet mode with: ./configure --disable-wallet -In this case there is no dependency on Berkeley DB 4.8 and SQLite. +In this case there is no dependency on SQLite or Berkeley DB. Mining is also possible in disable-wallet mode using the `getblocktemplate` RPC call. diff --git a/doc/cjdns.md b/doc/cjdns.md new file mode 100644 index 0000000000..5b2bcaf874 --- /dev/null +++ b/doc/cjdns.md @@ -0,0 +1,95 @@ +# CJDNS support in Bitcoin Core + +It is possible to run Bitcoin Core over CJDNS, an encrypted IPv6 network that +uses public-key cryptography for address allocation and a distributed hash table +for routing. + +## What is CJDNS? + +CJDNS is like a distributed, shared VPN with multiple entry points where every +participant can reach any other participant. All participants use addresses from +the `fc00::/8` network (reserved IPv6 range). Installation and configuration is +done outside of Bitcoin Core, similarly to a VPN (either in the host/OS or on +the network router). + +Compared to IPv4/IPv6, CJDNS provides end-to-end encryption and protects nodes +from traffic analysis and filtering. + +Used with Tor and I2P, CJDNS is a complementary option that can enhance network +redundancy and robustness for both the Bitcoin network and individual nodes. + +Each network has different characteristics. For instance, Tor is widely used but +somewhat centralized. I2P connections have a source address and I2P is slow. +CJDNS is fast but does not hide the sender and the recipient from intermediate +routers. + +## Installing CJDNS and connecting to the network + +To install and set up CJDNS, follow the instructions at +https://github.com/cjdelisle/cjdns#cjdns. + +Don't skip steps +["2. Find a friend"](https://github.com/cjdelisle/cjdns#2-find-a-friend) and +["3. Connect your node to your friend's +node"](https://github.com/cjdelisle/cjdns#3-connect-your-node-to-your-friends-node). +You need to be connected to the CJDNS network before it will work with your +Bitcoin Core node. + +Typically, CJDNS might be launched from its directory with +`sudo ./cjdroute < cjdroute.conf` and it sheds permissions after setting up the +[TUN](https://en.wikipedia.org/wiki/TUN/TAP) interface. You may also [launch it as an +unprivileged user](https://github.com/cjdelisle/cjdns/blob/master/doc/non-root-user.md) +with some additional setup. + +The network connection can be checked by running `./tools/peerStats` from the +CJDNS directory. + +## Run Bitcoin Core with CJDNS + +Once you are connected to the CJDNS network, the following Bitcoin Core +configuration option makes CJDNS peers automatically reachable: + +``` +-cjdnsreachable +``` + +When enabled, this option tells Bitcoin Core that it is running in an +environment where a connection to an `fc00::/8` address will be to the CJDNS +network instead of to an [RFC4193](https://datatracker.ietf.org/doc/html/rfc4193) +IPv6 local network. This helps Bitcoin Core perform better address management: + - Your node can consider incoming `fc00::/8` connections to be from the CJDNS + network rather than from an IPv6 private one. + - If one of your node's local addresses is `fc00::/8`, then it can choose to + gossip that address to peers. + +## Additional configuration options related to CJDNS + +``` +-onlynet=cjdns +``` + +Make automatic outbound connections only to CJDNS addresses. Inbound and manual +connections are not affected by this option. It can be specified multiple times +to allow multiple networks, e.g. onlynet=cjdns, onlynet=i2p, onlynet=onion. + +CJDNS support was added to Bitcoin Core in version 23.0 and there may be fewer +CJDNS peers than Tor or IP ones. You can use `bitcoin-cli -addrinfo` to see the +number of CJDNS addresses known to your node. + +In general, a node can be run with both an onion service and CJDNS (or any/all +of IPv4/IPv6/onion/I2P/CJDNS), which can provide a potential fallback if one of +the networks has issues. There are a number of ways to configure this; see +[doc/tor.md](https://github.com/bitcoin/bitcoin/blob/master/doc/tor.md) for +details. + +## CJDNS-related information in Bitcoin Core + +There are several ways to see your CJDNS address in Bitcoin Core: +- in the "Local addresses" output of CLI `-netinfo` +- in the "localaddresses" output of RPC `getnetworkinfo` + +To see which CJDNS peers your node is connected to, use `bitcoin-cli -netinfo 4` +or the `getpeerinfo` RPC (i.e. `bitcoin-cli getpeerinfo`). + +To see which CJDNS addresses your node knows, use the `getnodeaddresses 0 cjdns` +RPC. diff --git a/doc/dependencies.md b/doc/dependencies.md index 21af338119..57d93032d4 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -1,48 +1,49 @@ -Dependencies -============ - -These are the dependencies currently used by Bitcoin Core. You can find instructions for installing them in the `build-*.md` file for your platform. - -| Dependency | Version used | Minimum required | CVEs | Shared | [Bundled Qt library](https://doc.qt.io/qt-5/configure-options.html#third-party-libraries) | -| --- | --- | --- | --- | --- | --- | -| Berkeley DB | [4.8.30](https://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html) | 4.8.x | No | | | -| Boost | [1.77.0](https://www.boost.org/users/download/) | [1.64.0](https://github.com/bitcoin/bitcoin/pull/22320) | No | | | -| Clang<sup>[ \* ](#note1)</sup> | | [7.0](https://releases.llvm.org/download.html) (C++17 & std::filesystem support) | | | | -| Fontconfig | [2.12.6](https://www.freedesktop.org/software/fontconfig/release/) | | No | Yes | | -| FreeType | [2.11.0](https://download.savannah.gnu.org/releases/freetype) | | No | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Android only) | -| GCC | | [8.1](https://gcc.gnu.org/) (C++17 & std::filesystem support) | | | | -| glibc | | [2.18](https://www.gnu.org/software/libc/) | | | | | -| HarfBuzz-NG | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | -| libevent | [2.1.12-stable](https://github.com/libevent/libevent/releases) | [2.0.21](https://github.com/bitcoin/bitcoin/pull/18676) | No | | | -| libnatpmp | git commit [4536032...](https://github.com/miniupnp/libnatpmp/tree/4536032ae32268a45c073a4d5e91bbab4534773a) | | No | | | -| libpng | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | -| MiniUPnPc | [2.2.2](https://miniupnp.tuxfamily.org/files) | | No | | | -| PCRE | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | -| Python (tests) | | [3.6](https://www.python.org/downloads) | | | | -| qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | | -| Qt | [5.15.2](https://download.qt.io/official_releases/qt/) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | | | -| SQLite | [3.32.1](https://sqlite.org/download.html) | [3.7.17](https://github.com/bitcoin/bitcoin/pull/19077) | | | | -| XCB | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | -| systemtap ([tracing](tracing.md))| [4.5](https://sourceware.org/systemtap/ftp/releases/) | | | | | -| xkbcommon | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | -| ZeroMQ | [4.3.1](https://github.com/zeromq/libzmq/releases) | 4.0.0 | No | | | -| zlib | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | - -<a name="note1">Note \*</a> : When compiling with `-stdlib=libc++`, the minimum supported libc++ version is 7.0. - -Controlling dependencies ------------------------- -Some dependencies are not needed in all configurations. The following are some factors that affect the dependency list. - -#### Options passed to `./configure` -* MiniUPnPc is not needed with `--without-miniupnpc`. -* libnatpmp is not needed with `--without-natpmp`. -* Berkeley DB is not needed with `--disable-wallet` or `--without-bdb`. -* SQLite is not needed with `--disable-wallet` or `--without-sqlite`. -* Qt is not needed with `--without-gui`. -* If the qrencode dependency is absent, QR support won't be added. To force an error when that happens, pass `--with-qrencode`. -* If the systemtap dependency is absent, USDT support won't compiled in. -* ZeroMQ is needed only with the `--with-zmq` option. - -#### Other -* Not-Qt-bundled zlib is required to build the [DMG tool](../contrib/macdeploy/README.md#deterministic-macos-dmg-notes) from the libdmg-hfsplus project. +# Dependencies + +These are the dependencies used by Bitcoin Core. +You can find installation instructions in the `build-*.md` file for your platform. +"Runtime" and "Version Used" are both in reference to the release binaries. + +| Dependency | Minimum required | +| --- | --- | +| [Autoconf](https://www.gnu.org/software/autoconf/) | [2.69](https://github.com/bitcoin/bitcoin/pull/17769) | +| [Automake](https://www.gnu.org/software/automake/) | [1.13](https://github.com/bitcoin/bitcoin/pull/18290) | +| [Clang](https://clang.llvm.org) | [8.0](https://github.com/bitcoin/bitcoin/pull/24164) | +| [GCC](https://gcc.gnu.org) | [8.1](https://github.com/bitcoin/bitcoin/pull/23060) | +| [Python](https://www.python.org) (tests) | [3.6](https://github.com/bitcoin/bitcoin/pull/19504) | +| [systemtap](https://sourceware.org/systemtap/) ([tracing](tracing.md))| N/A | + +## Required + +| Dependency | Version used | Minimum required | Runtime | +| --- | --- | --- | --- | +| [Boost](https://www.boost.org/users/download/) | 1.77.0 | [1.64.0](https://github.com/bitcoin/bitcoin/pull/22320) | No | +| [libevent](https://github.com/libevent/libevent/releases) | 2.1.12-stable | [2.0.21](https://github.com/bitcoin/bitcoin/pull/18676) | No | +| [glibc](https://www.gnu.org/software/libc/) | N/A | [2.18](https://github.com/bitcoin/bitcoin/pull/23511) | Yes | + +## Optional + +### GUI +| Dependency | Version used | Minimum required | Runtime | +| --- | --- | --- | --- | +| [Fontconfig](https://www.freedesktop.org/wiki/Software/fontconfig/) | 2.12.6 | 2.6 | Yes | +| [FreeType](https://freetype.org) | 2.11.0 | 2.3.0 | Yes | +| [qrencode](https://fukuchi.org/works/qrencode/) | [3.4.4](https://fukuchi.org/works/qrencode) | | No | +| [Qt](https://www.qt.io) | [5.15.3](https://download.qt.io/official_releases/qt/) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | + +### Networking +| Dependency | Version used | Minimum required | Runtime | +| --- | --- | --- | --- | +| [libnatpmp](https://github.com/miniupnp/libnatpmp/) | commit [4536032...](https://github.com/miniupnp/libnatpmp/tree/4536032ae32268a45c073a4d5e91bbab4534773a) | | No | +| [MiniUPnPc](https://miniupnp.tuxfamily.org/) | 2.2.2 | 1.9 | No | + +### Notifications +| Dependency | Version used | Minimum required | Runtime | +| --- | --- | --- | --- | +| [ZeroMQ](https://zeromq.org) | 4.3.4 | 4.0.0 | No | + +### Wallet +| Dependency | Version used | Minimum required | Runtime | +| --- | --- | --- | --- | +| [Berkeley DB](https://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html) (legacy wallet) | 4.8.30 | 4.8.x | No | +| [SQLite](https://sqlite.org) | 3.32.1 | [3.7.17](https://github.com/bitcoin/bitcoin/pull/19077) | No | diff --git a/doc/developer-notes.md b/doc/developer-notes.md index bfb64093e1..c3ab3fa953 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -137,11 +137,57 @@ public: } // namespace foo ``` +Coding Style (C++ named arguments) +------------------------------ + +When passing named arguments, use a format that clang-tidy understands. The +argument names can otherwise not be verified by clang-tidy. + +For example: + +```c++ +void function(Addrman& addrman, bool clear); + +int main() +{ + function(g_addrman, /*clear=*/false); +} +``` + +### Running clang-tidy + +To run clang-tidy on Ubuntu/Debian, install the dependencies: + +```sh +apt install clang-tidy bear clang +``` + +Then, pass clang as compiler to configure, and use bear to produce the `compile_commands.json`: + +```sh +./autogen.sh && ./configure CC=clang CXX=clang++ +make clean && bear make -j $(nproc) # For bear 2.x +make clean && bear -- make -j $(nproc) # For bear 3.x +``` + +To run clang-tidy on all source files: + +```sh +( cd ./src/ && run-clang-tidy -j $(nproc) ) +``` + +To run clang-tidy on the changed source lines: + +```sh +git diff | ( cd ./src/ && clang-tidy-diff -p2 -j $(nproc) ) +``` + Coding Style (Python) --------------------- Refer to [/test/functional/README.md#style-guidelines](/test/functional/README.md#style-guidelines). + Coding Style (Doxygen-compatible comments) ------------------------------------------ diff --git a/doc/i2p.md b/doc/i2p.md index e45b5efb9b..39f65c4e5f 100644 --- a/doc/i2p.md +++ b/doc/i2p.md @@ -80,15 +80,15 @@ phase when syncing up a new node can be very slow. This phase can be sped up by using other networks, for instance `onlynet=onion`, at the same time. In general, a node can be run with both onion and I2P hidden services (or -any/all of IPv4/IPv6/onion/I2P), which can provide a potential fallback if one -of the networks has issues. +any/all of IPv4/IPv6/onion/I2P/CJDNS), which can provide a potential fallback if +one of the networks has issues. ## I2P-related information in Bitcoin Core There are several ways to see your I2P address in Bitcoin Core: -- in the debug log (grep for `AddLocal`, the I2P address ends in `.b32.i2p`) -- in the output of the `getnetworkinfo` RPC in the "localaddresses" section -- in the output of `bitcoin-cli -netinfo` peer connections dashboard +- in the "Local addresses" output of CLI `-netinfo` +- in the "localaddresses" output of RPC `getnetworkinfo` +- in the debug log (grep for `AddLocal`; the I2P address ends in `.b32.i2p`) To see which I2P peers your node is connected to, use `bitcoin-cli -netinfo 4` or the `getpeerinfo` RPC (e.g. `bitcoin-cli getpeerinfo`). diff --git a/doc/multisig-tutorial.md b/doc/multisig-tutorial.md index 0793040418..1d2b3244bc 100644 --- a/doc/multisig-tutorial.md +++ b/doc/multisig-tutorial.md @@ -29,7 +29,7 @@ These three wallets should not be used directly for privacy reasons (public key ```bash for ((n=1;n<=3;n++)) do - ./src/bitcoin-cli -signet -named createwallet wallet_name="participant_${n}" descriptors=true + ./src/bitcoin-cli -signet createwallet "participant_${n}" done ``` @@ -109,7 +109,7 @@ Then import the descriptors created in the previous step using the `importdescri After that, `getwalletinfo` can be used to check if the wallet was created successfully. ```bash -./src/bitcoin-cli -signet -named createwallet wallet_name="multisig_wallet_01" disable_private_keys=true blank=true descriptors=true +./src/bitcoin-cli -signet -named createwallet wallet_name="multisig_wallet_01" disable_private_keys=true blank=true ./src/bitcoin-cli -signet -rpcwallet="multisig_wallet_01" importdescriptors "$multisig_desc" @@ -238,4 +238,4 @@ psbt_2=$(./src/bitcoin-cli -signet -rpcwallet="participant_2" walletprocesspsbt finalized_psbt_hex=$(./src/bitcoin-cli -signet finalizepsbt $psbt_2 | jq -r '.hex') ./src/bitcoin-cli -signet sendrawtransaction $finalized_psbt_hex -```
\ No newline at end of file +``` diff --git a/doc/release-notes-24118.md b/doc/release-notes-24118.md new file mode 100644 index 0000000000..16f23c7d00 --- /dev/null +++ b/doc/release-notes-24118.md @@ -0,0 +1,10 @@ +New RPCs +-------- + +- The `sendall` RPC spends specific UTXOs to one or more recipients + without creating change. By default, the `sendall` RPC will spend + every UTXO in the wallet. `sendall` is useful to empty wallets or to + create a changeless payment from select UTXOs. When creating a payment + from a specific amount for which the recipient incurs the transaction + fee, continue to use the `subtractfeefromamount` option via the + `send`, `sendtoaddress`, or `sendmany` RPCs. (#24118) diff --git a/doc/release-notes-24494.md b/doc/release-notes-24494.md new file mode 100644 index 0000000000..afbb926433 --- /dev/null +++ b/doc/release-notes-24494.md @@ -0,0 +1,2 @@ +To help prevent fingerprinting transactions created by the Bitcoin Core wallet, change output +amounts are now randomized. (#24494) diff --git a/doc/release-notes-empty-template.md b/doc/release-notes-empty-template.md new file mode 100644 index 0000000000..8462714898 --- /dev/null +++ b/doc/release-notes-empty-template.md @@ -0,0 +1,99 @@ +*The release notes draft is a temporary file that can be added to by anyone. See +[/doc/developer-notes.md#release-notes](/doc/developer-notes.md#release-notes) +for the process.* + +*version* Release Notes Draft +=============================== + +Bitcoin Core version *version* is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-*version*/> + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +P2P and network changes +----------------------- + +Updated RPCs +------------ + + +Changes to wallet related RPCs can be found in the Wallet section below. + +New RPCs +-------- + +Build System +------------ + +Updated settings +---------------- + + +Changes to GUI or wallet related settings can be found in the GUI or Wallet section below. + +New settings +------------ + +Tools and Utilities +------------------- + +Wallet +------ + +GUI changes +----------- + +Low-level changes +================= + +RPC +--- + +Tests +----- + +*version* change log +==================== + +Credits +======= + +Thanks to everyone who directly contributed to this release: + + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes.md b/doc/release-notes.md index 2342342ae2..8462714898 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -1,17 +1,7 @@ -*After branching off for a major version release of Bitcoin Core, use this -template to create the initial release notes draft.* - *The release notes draft is a temporary file that can be added to by anyone. See [/doc/developer-notes.md#release-notes](/doc/developer-notes.md#release-notes) for the process.* -*Create the draft, named* "*version* Release Notes Draft" -*(e.g. "23.0 Release Notes Draft"), as a collaborative wiki in:* - -https://github.com/bitcoin-core/bitcoin-devwiki/wiki/ - -*Before the final release, move the notes back to this git repository.* - *version* Release Notes Draft =============================== @@ -54,9 +44,51 @@ unsupported systems. Notable changes =============== -Example item +P2P and network changes +----------------------- + +Updated RPCs +------------ + + +Changes to wallet related RPCs can be found in the Wallet section below. + +New RPCs +-------- + +Build System +------------ + +Updated settings +---------------- + + +Changes to GUI or wallet related settings can be found in the GUI or Wallet section below. + +New settings ------------ +Tools and Utilities +------------------- + +Wallet +------ + +GUI changes +----------- + +Low-level changes +================= + +RPC +--- + +Tests +----- + +*version* change log +==================== + Credits ======= diff --git a/doc/release-process.md b/doc/release-process.md index 5a74f72b6e..bc80e3d072 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -47,13 +47,15 @@ Release Process #### After branch-off (on the major release branch) - Update the versions. +- Create the draft, named "*version* Release Notes Draft", as a [collaborative wiki](https://github.com/bitcoin-core/bitcoin-devwiki/wiki/_new). +- Clear the release notes: `cp doc/release-notes-empty-template.md doc/release-notes.md` - Create a pinned meta-issue for testing the release candidate (see [this issue](https://github.com/bitcoin/bitcoin/issues/17079) for an example) and provide a link to it in the release announcements where useful. - Translations on Transifex - Change the auto-update URL for the new major version's resource away from `master` and to the branch, e.g. `https://raw.githubusercontent.com/bitcoin/bitcoin/<branch>/src/qt/locale/bitcoin_en.xlf`. Do not forget this or it will keep tracking the translations on master instead, drifting away from the specific major release. #### Before final release -- Merge the release notes from the wiki into the branch. +- Merge the release notes from [the wiki](https://github.com/bitcoin-core/bitcoin-devwiki/wiki/) into the branch. - Ensure the "Needs release note" label is removed from all relevant pull requests and issues. #### Tagging a release (candidate) @@ -110,28 +112,24 @@ against other `guix-attest` signatures. git -C ./guix.sigs pull ``` -### Create the macOS SDK tarball: (first time, or when SDK version changes) +### Create the macOS SDK tarball (first time, or when SDK version changes) Create the macOS SDK tarball, see the [macdeploy instructions](/contrib/macdeploy/README.md#deterministic-macos-dmg-notes) for details. -### Build and attest to build outputs: +### Build and attest to build outputs Follow the relevant Guix README.md sections: - [Building](/contrib/guix/README.md#building) - [Attesting to build outputs](/contrib/guix/README.md#attesting-to-build-outputs) -### Verify other builders' signatures to your own. (Optional) +### Verify other builders' signatures to your own (optional) -Add other builders keys to your gpg keyring, and/or refresh keys: See `../bitcoin/contrib/builder-keys/README.md`. - -Follow the relevant Guix README.md sections: +- [Add other builders keys to your gpg keyring, and/or refresh keys](/contrib/builder-keys/README.md) - [Verifying build output attestations](/contrib/guix/README.md#verifying-build-output-attestations) -### Next steps: - -Commit your signature to guix.sigs: +### Commit your non codesigned signature to guix.sigs ```sh pushd ./guix.sigs @@ -141,29 +139,27 @@ git push # Assuming you can push to the guix.sigs tree popd ``` -Codesigner only: Create Windows/macOS detached signatures: -- Only one person handles codesigning. Everyone else should skip to the next step. -- Only once the Windows/macOS builds each have 3 matching signatures may they be signed with their respective release keys. +## Codesigning -Codesigner only: Sign the macOS binary: +### macOS codesigner only: Create detached macOS signatures (assuming [signapple](https://github.com/achow101/signapple/) is installed and up to date with master branch) - transfer bitcoin-osx-unsigned.tar.gz to macOS for signing tar xf bitcoin-osx-unsigned.tar.gz - ./detached-sig-create.sh -s "Key ID" + ./detached-sig-create.sh /path/to/codesign.p12 Enter the keychain password and authorize the signature - Move signature-osx.tar.gz back to the guix-build host + signature-osx.tar.gz will be created -Codesigner only: Sign the windows binaries: +### Windows codesigner only: Create detached Windows signatures tar xf bitcoin-win-unsigned.tar.gz ./detached-sig-create.sh -key /path/to/codesign.key Enter the passphrase for the key when prompted signature-win.tar.gz will be created -Code-signer only: It is advised to test that the code signature attaches properly prior to tagging by performing the `guix-codesign` step. +### Windows and macOS codesigners only: test code signatures +It is advised to test that the code signature attaches properly prior to tagging by performing the `guix-codesign` step. However if this is done, once the release has been tagged in the bitcoin-detached-sigs repo, the `guix-codesign` step must be performed again in order for the guix attestation to be valid when compared against the attestations of non-codesigner builds. -Codesigner only: Commit the detached codesign payloads: +### Windows and macOS codesigners only: Commit the detached codesign payloads ```sh pushd ./bitcoin-detached-sigs @@ -178,16 +174,21 @@ git push the current branch and new tag popd ``` -Non-codesigners: wait for Windows/macOS detached signatures: +### Non-codesigners: wait for Windows and macOS detached signatures -- Once the Windows/macOS builds each have 3 matching signatures, they will be signed with their respective release keys. +- Once the Windows and macOS builds each have 3 matching signatures, they will be signed with their respective release keys. - Detached signatures will then be committed to the [bitcoin-detached-sigs](https://github.com/bitcoin-core/bitcoin-detached-sigs) repository, which can be combined with the unsigned apps to create signed binaries. -Create (and optionally verify) the codesigned outputs: +### Create the codesigned build outputs -- [Codesigning](/contrib/guix/README.md#codesigning) +- [Codesigning build outputs](/contrib/guix/README.md#codesigning-build-outputs) + +### Verify other builders' signatures to your own (optional) + +- [Add other builders keys to your gpg keyring, and/or refresh keys](/contrib/builder-keys/README.md) +- [Verifying build output attestations](/contrib/guix/README.md#verifying-build-output-attestations) -Commit your signature for the signed macOS/Windows binaries: +### Commit your codesigned signature to guix.sigs (for the signed macOS/Windows binaries) ```sh pushd ./guix.sigs @@ -197,7 +198,7 @@ git push # Assuming you can push to the guix.sigs tree popd ``` -### After 3 or more people have guix-built and their results match: +## After 3 or more people have guix-built and their results match Combine the `all.SHA256SUMS.asc` file from all signers into `SHA256SUMS.asc`: diff --git a/doc/tor.md b/doc/tor.md index b7c4f7d425..08d031d084 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -16,9 +16,9 @@ configure Tor. ## How to see information about your Tor configuration via Bitcoin Core There are several ways to see your local onion address in Bitcoin Core: -- in the debug log (grep for "tor:" or "AddLocal") -- in the output of RPC `getnetworkinfo` in the "localaddresses" section -- in the output of the CLI `-netinfo` peer connections dashboard +- in the "Local addresses" output of CLI `-netinfo` +- in the "localaddresses" output of RPC `getnetworkinfo` +- in the debug log (grep for "AddLocal"; the Tor address ends in `.onion`) You may set the `-debug=tor` config logging option to have additional information in the debug log about your Tor configuration. @@ -27,6 +27,9 @@ CLI `-addrinfo` returns the number of addresses known to your node per network. This can be useful to see how many onion peers your node knows, e.g. for `-onlynet=onion`. +To fetch a number of onion addresses that your node knows, for example seven +addresses, use the `getnodeaddresses 7 onion` RPC. + ## 1. Run Bitcoin Core behind a Tor proxy The first step is running Bitcoin Core behind a Tor proxy. This will already anonymize all @@ -58,7 +61,7 @@ outgoing connections, but more is possible. -onlynet=onion Make automatic outbound connections only to .onion addresses. Inbound and manual connections are not affected by this option. It can be specified multiple times to allow multiple networks, - e.g. onlynet=onion, onlynet=i2p. + e.g. onlynet=onion, onlynet=i2p, onlynet=cjdns. In a typical situation, this suffices to run behind a Tor proxy: diff --git a/src/Makefile.am b/src/Makefile.am index 8f4cbee62f..12e4c7d8b7 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -8,9 +8,9 @@ print-%: FORCE DIST_SUBDIRS = secp256k1 -AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) $(LTO_LDFLAGS) -AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) $(LTO_CXXFLAGS) -AM_CPPFLAGS = $(DEBUG_CPPFLAGS) $(HARDENED_CPPFLAGS) +AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) $(LTO_LDFLAGS) $(CORE_LDFLAGS) +AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) $(LTO_CXXFLAGS) $(CORE_CXXFLAGS) +AM_CPPFLAGS = $(DEBUG_CPPFLAGS) $(HARDENED_CPPFLAGS) $(CORE_CPPFLAGS) AM_LIBTOOLFLAGS = --preserve-dup-deps PTHREAD_FLAGS = $(PTHREAD_CFLAGS) $(PTHREAD_LIBS) EXTRA_LIBRARIES = @@ -209,6 +209,7 @@ BITCOIN_CORE_H = \ reverse_iterator.h \ rpc/blockchain.h \ rpc/client.h \ + rpc/mempool.h \ rpc/mining.h \ rpc/protocol.h \ rpc/rawtransaction_util.h \ @@ -220,6 +221,7 @@ BITCOIN_CORE_H = \ scheduler.h \ script/descriptor.h \ script/keyorigin.h \ + script/miniscript.h \ script/sigcache.h \ script/sign.h \ script/signingprovider.h \ @@ -370,12 +372,14 @@ libbitcoin_node_a_SOURCES = \ pow.cpp \ rest.cpp \ rpc/blockchain.cpp \ + rpc/mempool.cpp \ rpc/mining.cpp \ rpc/misc.cpp \ rpc/net.cpp \ rpc/rawtransaction.cpp \ rpc/server.cpp \ rpc/server_util.cpp \ + rpc/txoutproof.cpp \ script/sigcache.cpp \ shutdown.cpp \ signet.cpp \ @@ -587,6 +591,7 @@ libbitcoin_common_a_SOURCES = \ rpc/util.cpp \ scheduler.cpp \ script/descriptor.cpp \ + script/miniscript.cpp \ script/sign.cpp \ script/signingprovider.cpp \ script/standard.cpp \ @@ -603,7 +608,6 @@ libbitcoin_util_a_SOURCES = \ chainparamsbase.cpp \ clientversion.cpp \ compat/glibcxx_sanity.cpp \ - compat/strnlen.cpp \ fs.cpp \ interfaces/echo.cpp \ interfaces/handler.cpp \ @@ -618,6 +622,7 @@ libbitcoin_util_a_SOURCES = \ util/asmap.cpp \ util/bip32.cpp \ util/bytevectorhash.cpp \ + util/check.cpp \ util/error.cpp \ util/fees.cpp \ util/getuniquepath.cpp \ @@ -836,6 +841,7 @@ bitcoin_chainstate_SOURCES = \ uint256.cpp \ util/asmap.cpp \ util/bytevectorhash.cpp \ + util/check.cpp \ util/getuniquepath.cpp \ util/hasher.cpp \ util/moneystr.cpp \ diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 0bcce6ebe1..5dae4374e3 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -13,38 +13,39 @@ GENERATED_BENCH_FILES = $(RAW_BENCH_FILES:.raw=.raw.h) bench_bench_bitcoin_SOURCES = \ $(RAW_BENCH_FILES) \ bench/addrman.cpp \ - bench/bench_bitcoin.cpp \ + bench/base58.cpp \ + bench/bech32.cpp \ bench/bench.cpp \ bench/bench.h \ + bench/bench_bitcoin.cpp \ bench/block_assemble.cpp \ + bench/ccoins_caching.cpp \ + bench/chacha20.cpp \ + bench/chacha_poly_aead.cpp \ bench/checkblock.cpp \ bench/checkqueue.cpp \ - bench/data.h \ + bench/crypto_hash.cpp \ bench/data.cpp \ + bench/data.h \ bench/duplicate_inputs.cpp \ bench/examples.cpp \ - bench/rollingbloom.cpp \ - bench/chacha20.cpp \ - bench/chacha_poly_aead.cpp \ - bench/crypto_hash.cpp \ - bench/ccoins_caching.cpp \ bench/gcs_filter.cpp \ bench/hashpadding.cpp \ - bench/merkle_root.cpp \ + bench/lockedpool.cpp \ + bench/logging.cpp \ bench/mempool_eviction.cpp \ bench/mempool_stress.cpp \ - bench/nanobench.h \ + bench/merkle_root.cpp \ bench/nanobench.cpp \ + bench/nanobench.h \ bench/peer_eviction.cpp \ + bench/poly1305.cpp \ + bench/prevector.cpp \ + bench/rollingbloom.cpp \ bench/rpc_blockchain.cpp \ bench/rpc_mempool.cpp \ bench/util_time.cpp \ - bench/verify_script.cpp \ - bench/base58.cpp \ - bench/bech32.cpp \ - bench/lockedpool.cpp \ - bench/poly1305.cpp \ - bench/prevector.cpp + bench/verify_script.cpp nodist_bench_bench_bitcoin_SOURCES = $(GENERATED_BENCH_FILES) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index e2e08468a7..9ae7886a6e 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -103,6 +103,7 @@ BITCOIN_TESTS =\ test/merkle_tests.cpp \ test/merkleblock_tests.cpp \ test/miner_tests.cpp \ + test/miniscript_tests.cpp \ test/minisketch_tests.cpp \ test/multisig_tests.cpp \ test/net_peer_eviction_tests.cpp \ @@ -121,6 +122,7 @@ BITCOIN_TESTS =\ test/scheduler_tests.cpp \ test/script_p2sh_tests.cpp \ test/script_parse_tests.cpp \ + test/script_segwit_tests.cpp \ test/script_standard_tests.cpp \ test/script_tests.cpp \ test/scriptnum10.h \ @@ -175,8 +177,11 @@ if USE_BDB BITCOIN_TESTS += wallet/test/db_tests.cpp endif -if USE_SQLITE FUZZ_WALLET_SRC = \ + wallet/test/fuzz/coinselection.cpp + +if USE_SQLITE +FUZZ_WALLET_SRC += \ wallet/test/fuzz/notifications.cpp endif # USE_SQLITE @@ -262,6 +267,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/locale.cpp \ test/fuzz/merkleblock.cpp \ test/fuzz/message.cpp \ + test/fuzz/miniscript_decode.cpp \ test/fuzz/minisketch.cpp \ test/fuzz/muhash.cpp \ test/fuzz/multiplication_overflow.cpp \ @@ -384,8 +390,19 @@ univalue_test_object_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) endif %.cpp.test: %.cpp - @echo Running tests: `cat $< | grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1` from $< - $(AM_V_at)$(TEST_BINARY) --catch_system_errors=no -l test_suite -t "`cat $< | grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1`" -- DEBUG_LOG_OUT > $<.log 2>&1 || (cat $<.log && false) + @echo Running tests: $$(\ + cat $< | \ + grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | \ + cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1\ + ) from $< + $(AM_V_at)export TEST_LOGFILE=$(abs_builddir)/$$(\ + echo $< | grep -E -o "(wallet/test/.*\.cpp|test/.*\.cpp)" | $(SED) -e s/\.cpp/.log/ \ + ) && \ + $(TEST_BINARY) --catch_system_errors=no -l test_suite -t "$$(\ + cat $< | \ + grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | \ + cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1\ + )" -- DEBUG_LOG_OUT > "$$TEST_LOGFILE" 2>&1 || (cat "$$TEST_LOGFILE" && false) %.json.h: %.json @$(MKDIR_P) $(@D) diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 0fa8f3c3da..1fa2644647 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -185,7 +185,7 @@ void ReadFromStream(AddrMan& addr, CDataStream& ssPeers) std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman) { auto check_addrman = std::clamp<int32_t>(args.GetIntArg("-checkaddrman", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), 0, 1000000); - addrman = std::make_unique<AddrMan>(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); + addrman = std::make_unique<AddrMan>(asmap, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); int64_t nStart = GetTimeMillis(); const auto path_addr{args.GetDataDirNet() / "peers.dat"}; @@ -194,7 +194,7 @@ std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const A LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman->size(), GetTimeMillis() - nStart); } catch (const DbNotFoundError&) { // Addrman can be in an inconsistent state after failure, reset it - addrman = std::make_unique<AddrMan>(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); + addrman = std::make_unique<AddrMan>(asmap, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); LogPrintf("Creating peers.dat because the file was not found (%s)\n", fs::quoted(fs::PathToString(path_addr))); DumpPeerAddresses(args, *addrman); } catch (const InvalidAddrManVersionError&) { @@ -203,7 +203,7 @@ std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const A return strprintf(_("Failed to rename invalid peers.dat file. Please move or delete it and try again.")); } // Addrman can be in an inconsistent state after failure, reset it - addrman = std::make_unique<AddrMan>(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); + addrman = std::make_unique<AddrMan>(asmap, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); LogPrintf("Creating new peers.dat because the file version was not compatible (%s). Original backed up to peers.dat.bak\n", fs::quoted(fs::PathToString(path_addr))); DumpPeerAddresses(args, *addrman); } catch (const std::exception& e) { diff --git a/src/addrman.cpp b/src/addrman.cpp index 2fd8143c1c..2a08d99eef 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -946,16 +946,16 @@ std::optional<AddressPosition> AddrManImpl::FindAddressEntry_(const CAddress& ad if(addr_info->fInTried) { int bucket{addr_info->GetTriedBucket(nKey, m_asmap)}; - return AddressPosition(/*tried=*/true, - /*multiplicity=*/1, - /*bucket=*/bucket, - /*position=*/addr_info->GetBucketPosition(nKey, false, bucket)); + return AddressPosition(/*tried_in=*/true, + /*multiplicity_in=*/1, + /*bucket_in=*/bucket, + /*position_in=*/addr_info->GetBucketPosition(nKey, false, bucket)); } else { int bucket{addr_info->GetNewBucket(nKey, m_asmap)}; - return AddressPosition(/*tried=*/false, - /*multiplicity=*/addr_info->nRefCount, - /*bucket=*/bucket, - /*position=*/addr_info->GetBucketPosition(nKey, true, bucket)); + return AddressPosition(/*tried_in=*/false, + /*multiplicity_in=*/addr_info->nRefCount, + /*bucket_in=*/bucket, + /*position_in=*/addr_info->GetBucketPosition(nKey, true, bucket)); } } diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp index 3ca58b923e..34bc4380dd 100644 --- a/src/bench/addrman.cpp +++ b/src/bench/addrman.cpp @@ -101,7 +101,7 @@ static void AddrManGetAddr(benchmark::Bench& bench) FillAddrMan(addrman); bench.run([&] { - const auto& addresses = addrman.GetAddr(/* max_addresses */ 2500, /* max_pct */ 23, /* network */ std::nullopt); + const auto& addresses = addrman.GetAddr(/*max_addresses=*/2500, /*max_pct=*/23, /*network=*/std::nullopt); assert(addresses.size() > 0); }); } diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 609c592d20..de8ab9807c 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -13,7 +13,7 @@ using node::NodeContext; using wallet::AttemptSelection; -using wallet::CInputCoin; +using wallet::CHANGE_LOWER; using wallet::COutput; using wallet::CWallet; using wallet::CWalletTx; @@ -58,14 +58,22 @@ static void CoinSelection(benchmark::Bench& bench) // Create coins std::vector<COutput> coins; for (const auto& wtx : wtxs) { - coins.emplace_back(wallet, *wtx, 0 /* iIn */, 6 * 24 /* nDepthIn */, true /* spendable */, true /* solvable */, true /* safe */); + coins.emplace_back(COutPoint(wtx->GetHash(), 0), wtx->tx->vout.at(0), /*depth=*/6 * 24, GetTxSpendSize(wallet, *wtx, 0), /*spendable=*/true, /*solvable=*/true, /*safe=*/true, wtx->GetTxTime(), /*from_me=*/true); } const CoinEligibilityFilter filter_standard(1, 6, 0); - const CoinSelectionParams coin_selection_params(/* change_output_size= */ 34, - /* change_spend_size= */ 148, /* effective_feerate= */ CFeeRate(0), - /* long_term_feerate= */ CFeeRate(0), /* discard_feerate= */ CFeeRate(0), - /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); + FastRandomContext rand{}; + const CoinSelectionParams coin_selection_params{ + rand, + /*change_output_size=*/ 34, + /*change_spend_size=*/ 148, + /*min_change_target=*/ CHANGE_LOWER, + /*effective_feerate=*/ CFeeRate(0), + /*long_term_feerate=*/ CFeeRate(0), + /*discard_feerate=*/ CFeeRate(0), + /*tx_noinputs_size=*/ 0, + /*avoid_partial=*/ false, + }; bench.run([&] { auto result = AttemptSelection(wallet, 1003 * COIN, filter_standard, coins, coin_selection_params); assert(result); @@ -74,17 +82,15 @@ static void CoinSelection(benchmark::Bench& bench) }); } -typedef std::set<CInputCoin> CoinSet; - // Copied from src/wallet/test/coinselector_tests.cpp static void add_coin(const CAmount& nValue, int nInput, std::vector<OutputGroup>& set) { CMutableTransaction tx; tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; - CInputCoin coin(MakeTransactionRef(tx), nInput); + COutput output(COutPoint(tx.GetHash(), nInput), tx.vout.at(nInput), /*depth=*/ 0, /*input_bytes=*/ -1, /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, /*time=*/ 0, /*from_me=*/ true); set.emplace_back(); - set.back().Insert(coin, 0, true, 0, 0, false); + set.back().Insert(output, /*ancestors=*/ 0, /*descendants=*/ 0, /*positive_only=*/ false); } // Copied from src/wallet/test/coinselector_tests.cpp static CAmount make_hard_case(int utxos, std::vector<OutputGroup>& utxo_pool) diff --git a/src/bench/logging.cpp b/src/bench/logging.cpp new file mode 100644 index 0000000000..d28777df9e --- /dev/null +++ b/src/bench/logging.cpp @@ -0,0 +1,48 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <logging.h> +#include <test/util/setup_common.h> + + +static void Logging(benchmark::Bench& bench, const std::vector<const char*>& extra_args, const std::function<void()>& log) +{ + TestingSetup test_setup{ + CBaseChainParams::REGTEST, + extra_args, + }; + + bench.run([&] { log(); }); +} + +static void LoggingYoThreadNames(benchmark::Bench& bench) +{ + Logging(bench, {"-logthreadnames=1"}, [] { LogPrintf("%s\n", "test"); }); +} +static void LoggingNoThreadNames(benchmark::Bench& bench) +{ + Logging(bench, {"-logthreadnames=0"}, [] { LogPrintf("%s\n", "test"); }); +} +static void LoggingYoCategory(benchmark::Bench& bench) +{ + Logging(bench, {"-logthreadnames=0", "-debug=net"}, [] { LogPrint(BCLog::NET, "%s\n", "test"); }); +} +static void LoggingNoCategory(benchmark::Bench& bench) +{ + Logging(bench, {"-logthreadnames=0", "-debug=0"}, [] { LogPrint(BCLog::NET, "%s\n", "test"); }); +} +static void LoggingNoFile(benchmark::Bench& bench) +{ + Logging(bench, {"-nodebuglogfile", "-debug=1"}, [] { + LogPrintf("%s\n", "test"); + LogPrint(BCLog::NET, "%s\n", "test"); + }); +} + +BENCHMARK(LoggingYoThreadNames); +BENCHMARK(LoggingNoThreadNames); +BENCHMARK(LoggingYoCategory); +BENCHMARK(LoggingNoCategory); +BENCHMARK(LoggingNoFile); diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp index afa4618e1b..a58658c4f1 100644 --- a/src/bench/mempool_stress.cpp +++ b/src/bench/mempool_stress.cpp @@ -86,7 +86,7 @@ static void ComplexMemPool(benchmark::Bench& bench) if (bench.complexityN() > 1) { childTxs = static_cast<int>(bench.complexityN()); } - std::vector<CTransactionRef> ordered_coins = CreateOrderedCoins(det_rand, childTxs, /* min_ancestors */ 1); + std::vector<CTransactionRef> ordered_coins = CreateOrderedCoins(det_rand, childTxs, /*min_ancestors=*/1); const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(CBaseChainParams::MAIN); CTxMemPool pool; LOCK2(cs_main, pool.cs); @@ -103,7 +103,7 @@ static void MempoolCheck(benchmark::Bench& bench) { FastRandomContext det_rand{true}; const int childTxs = bench.complexityN() > 1 ? static_cast<int>(bench.complexityN()) : 2000; - const std::vector<CTransactionRef> ordered_coins = CreateOrderedCoins(det_rand, childTxs, /* min_ancestors */ 5); + const std::vector<CTransactionRef> ordered_coins = CreateOrderedCoins(det_rand, childTxs, /*min_ancestors=*/5); const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(CBaseChainParams::MAIN, {"-checkmempool=1"}); CTxMemPool pool; LOCK2(cs_main, pool.cs); @@ -111,7 +111,7 @@ static void MempoolCheck(benchmark::Bench& bench) for (auto& tx : ordered_coins) AddTx(tx, pool); bench.run([&]() NO_THREAD_SAFETY_ANALYSIS { - pool.check(coins_tip, /* spendheight */ 2); + pool.check(coins_tip, /*spendheight=*/2); }); } diff --git a/src/bench/rpc_mempool.cpp b/src/bench/rpc_mempool.cpp index 12dcff5844..6e322ba6aa 100644 --- a/src/bench/rpc_mempool.cpp +++ b/src/bench/rpc_mempool.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> -#include <rpc/blockchain.h> +#include <rpc/mempool.h> #include <txmempool.h> #include <univalue.h> @@ -29,11 +29,11 @@ static void RpcMempool(benchmark::Bench& bench) tx.vout[0].scriptPubKey = CScript() << OP_1 << OP_EQUAL; tx.vout[0].nValue = i; const CTransactionRef tx_r{MakeTransactionRef(tx)}; - AddTx(tx_r, /* fee */ i, pool); + AddTx(tx_r, /*fee=*/i, pool); } bench.run([&] { - (void)MempoolToJSON(pool, /*verbose*/ true); + (void)MempoolToJSON(pool, /*verbose=*/true); }); } diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index d4b8794c6d..a5dd2f3bb1 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -52,10 +52,10 @@ static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const b }); } -static void WalletBalanceDirty(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ true, /* add_mine */ true); } -static void WalletBalanceClean(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_mine */ true); } -static void WalletBalanceMine(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_mine */ true); } -static void WalletBalanceWatch(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_mine */ false); } +static void WalletBalanceDirty(benchmark::Bench& bench) { WalletBalance(bench, /*set_dirty=*/true, /*add_mine=*/true); } +static void WalletBalanceClean(benchmark::Bench& bench) { WalletBalance(bench, /*set_dirty=*/false, /*add_mine=*/true); } +static void WalletBalanceMine(benchmark::Bench& bench) { WalletBalance(bench, /*set_dirty=*/false, /*add_mine=*/true); } +static void WalletBalanceWatch(benchmark::Bench& bench) { WalletBalance(bench, /*set_dirty=*/false, /*add_mine=*/false); } BENCHMARK(WalletBalanceDirty); BENCHMARK(WalletBalanceClean); diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index 72b8fefcc7..fcbb6aacce 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -192,7 +192,7 @@ int main(int argc, char* argv[]) bool new_block; auto sc = std::make_shared<submitblock_StateCatcher>(block.GetHash()); RegisterSharedValidationInterface(sc); - bool accepted = chainman.ProcessNewBlock(chainparams, blockptr, /* force_processing */ true, /* new_block */ &new_block); + bool accepted = chainman.ProcessNewBlock(chainparams, blockptr, /*force_processing=*/true, /*new_block=*/&new_block); UnregisterSharedValidationInterface(sc); if (!new_block && accepted) { std::cerr << "duplicate" << std::endl; diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index b297081cab..0b40626595 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -750,7 +750,7 @@ static void MutateTx(CMutableTransaction& tx, const std::string& command, static void OutputTxJSON(const CTransaction& tx) { UniValue entry(UniValue::VOBJ); - TxToUniv(tx, uint256(), entry); + TxToUniv(tx, /*block_hash=*/uint256(), entry); std::string jsonOutput = entry.write(4); tfm::format(std::cout, "%s\n", jsonOutput); diff --git a/src/bitcoin-util.cpp b/src/bitcoin-util.cpp index b457e0b354..a2f3fca26c 100644 --- a/src/bitcoin-util.cpp +++ b/src/bitcoin-util.cpp @@ -22,8 +22,6 @@ #include <memory> #include <thread> -#include <boost/algorithm/string.hpp> - static const int CONTINUE_EXECUTION=-1; const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; diff --git a/src/chainparams.cpp b/src/chainparams.cpp index d3ae6f4cb2..c65a816a5f 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -9,6 +9,7 @@ #include <consensus/merkle.h> #include <deploymentinfo.h> #include <hash.h> // for signet block challenge hash +#include <script/interpreter.h> #include <util/system.h> #include <assert.h> @@ -65,7 +66,10 @@ public: consensus.signet_blocks = false; consensus.signet_challenge.clear(); consensus.nSubsidyHalvingInterval = 210000; - consensus.BIP16Exception = uint256S("0x00000000000002dc756eebf4f49723ed8d30cc28a5f108eb94b1ba88ac4f9c22"); + consensus.script_flag_exceptions.emplace( // BIP16 exception + uint256S("0x00000000000002dc756eebf4f49723ed8d30cc28a5f108eb94b1ba88ac4f9c22"), SCRIPT_VERIFY_NONE); + consensus.script_flag_exceptions.emplace( // Taproot exception + uint256S("0x0000000000000000000f14c35b2d841e986ab5441de8c585d5ffe55ea1e395ad"), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS); consensus.BIP34Height = 227931; consensus.BIP34Hash = uint256S("0x000000000000024b89b42a942fe0d9fea3bb44ab7bd1b19115dd6a759c0808b8"); consensus.BIP65Height = 388381; // 000000000000000004c2b624ed5d7756c508d90fd0da2c7c679febfa6c4735f0 @@ -184,7 +188,8 @@ public: consensus.signet_blocks = false; consensus.signet_challenge.clear(); consensus.nSubsidyHalvingInterval = 210000; - consensus.BIP16Exception = uint256S("0x00000000dd30457c001f4095d208cc1296b0eed002427aa599874af7a432b105"); + consensus.script_flag_exceptions.emplace( // BIP16 exception + uint256S("0x00000000dd30457c001f4095d208cc1296b0eed002427aa599874af7a432b105"), SCRIPT_VERIFY_NONE); consensus.BIP34Height = 21111; consensus.BIP34Hash = uint256S("0x0000000023b3a96d3484e5abb3755c413e7d41500f8e2a5c3f0dd01299cd8ef8"); consensus.BIP65Height = 581885; // 00000000007f6655f22f98e72ed80d8b06dc761d5da09df0fa1dc4be4f861eb6 @@ -323,7 +328,6 @@ public: consensus.signet_blocks = true; consensus.signet_challenge.assign(bin.begin(), bin.end()); consensus.nSubsidyHalvingInterval = 210000; - consensus.BIP16Exception = uint256{}; consensus.BIP34Height = 1; consensus.BIP34Hash = uint256{}; consensus.BIP65Height = 1; @@ -391,13 +395,12 @@ public: consensus.signet_blocks = false; consensus.signet_challenge.clear(); consensus.nSubsidyHalvingInterval = 150; - consensus.BIP16Exception = uint256(); consensus.BIP34Height = 1; // Always active unless overridden consensus.BIP34Hash = uint256(); consensus.BIP65Height = 1; // Always active unless overridden consensus.BIP66Height = 1; // Always active unless overridden consensus.CSVHeight = 1; // Always active unless overridden - consensus.SegwitHeight = 1; // Always active unless overridden + consensus.SegwitHeight = 0; // Always active unless overridden consensus.MinBIP9WarningHeight = 0; consensus.powLimit = uint256S("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); consensus.nPowTargetTimespan = 14 * 24 * 60 * 60; // two weeks diff --git a/src/compat.h b/src/compat.h index 237b881b11..3ec4ab53fd 100644 --- a/src/compat.h +++ b/src/compat.h @@ -80,10 +80,6 @@ typedef int32_t ssize_t; #endif #endif -#if HAVE_DECL_STRNLEN == 0 -size_t strnlen( const char *start, size_t max_len); -#endif // HAVE_DECL_STRNLEN - #ifndef WIN32 typedef void* sockopt_arg_type; #else diff --git a/src/compat/strnlen.cpp b/src/compat/strnlen.cpp deleted file mode 100644 index 93a034a664..0000000000 --- a/src/compat/strnlen.cpp +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif - -#include <cstring> - -#if HAVE_DECL_STRNLEN == 0 -size_t strnlen( const char *start, size_t max_len) -{ - const char *end = (const char *)memchr(start, '\0', max_len); - - return end ? (size_t)(end - start) : max_len; -} -#endif // HAVE_DECL_STRNLEN diff --git a/src/consensus/params.h b/src/consensus/params.h index 1ed5ca469f..0881f2e388 100644 --- a/src/consensus/params.h +++ b/src/consensus/params.h @@ -7,7 +7,9 @@ #define BITCOIN_CONSENSUS_PARAMS_H #include <uint256.h> + #include <limits> +#include <map> namespace Consensus { @@ -70,8 +72,13 @@ struct BIP9Deployment { struct Params { uint256 hashGenesisBlock; int nSubsidyHalvingInterval; - /* Block hash that is excepted from BIP16 enforcement */ - uint256 BIP16Exception; + /** + * Hashes of blocks that + * - are known to be consensus valid, and + * - buried in the chain, and + * - fail if the default script verify flags are applied. + */ + std::map<uint256, uint32_t> script_flag_exceptions; /** Block height and hash at which BIP34 becomes active */ int BIP34Height; uint256 BIP34Hash; diff --git a/src/core_io.h b/src/core_io.h index 69b9ac3ebd..aa1381c374 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -53,8 +53,7 @@ UniValue ValueFromAmount(const CAmount amount); std::string FormatScript(const CScript& script); std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags = 0); std::string SighashToStr(unsigned char sighash_type); -void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool include_hex, bool include_address = true); -void ScriptToUniv(const CScript& script, UniValue& out); -void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex = true, int serialize_flags = 0, const CTxUndo* txundo = nullptr, TxVerbosity verbosity = TxVerbosity::SHOW_DETAILS); +void ScriptToUniv(const CScript& script, UniValue& out, bool include_hex = true, bool include_address = false); +void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex = true, int serialize_flags = 0, const CTxUndo* txundo = nullptr, TxVerbosity verbosity = TxVerbosity::SHOW_DETAILS); #endif // BITCOIN_CORE_IO_H diff --git a/src/core_write.cpp b/src/core_write.cpp index c4b6b8d27e..cfd4cb1b49 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -19,6 +19,10 @@ #include <util/strencodings.h> #include <util/system.h> +#include <map> +#include <string> +#include <vector> + UniValue ValueFromAmount(const CAmount amount) { static_assert(COIN > 1); @@ -143,29 +147,28 @@ std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags) return HexStr(ssTx); } -void ScriptToUniv(const CScript& script, UniValue& out) -{ - ScriptPubKeyToUniv(script, out, /* include_hex */ true, /* include_address */ false); -} - -void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool include_hex, bool include_address) +void ScriptToUniv(const CScript& script, UniValue& out, bool include_hex, bool include_address) { CTxDestination address; - out.pushKV("asm", ScriptToAsmStr(scriptPubKey)); - out.pushKV("desc", InferDescriptor(scriptPubKey, DUMMY_SIGNING_PROVIDER)->ToString()); - if (include_hex) out.pushKV("hex", HexStr(scriptPubKey)); + out.pushKV("asm", ScriptToAsmStr(script)); + if (include_address) { + out.pushKV("desc", InferDescriptor(script, DUMMY_SIGNING_PROVIDER)->ToString()); + } + if (include_hex) { + out.pushKV("hex", HexStr(script)); + } std::vector<std::vector<unsigned char>> solns; - const TxoutType type{Solver(scriptPubKey, solns)}; + const TxoutType type{Solver(script, solns)}; - if (include_address && ExtractDestination(scriptPubKey, address) && type != TxoutType::PUBKEY) { + if (include_address && ExtractDestination(script, address) && type != TxoutType::PUBKEY) { out.pushKV("address", EncodeDestination(address)); } out.pushKV("type", GetTxnOutputType(type)); } -void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex, int serialize_flags, const CTxUndo* txundo, TxVerbosity verbosity) +void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex, int serialize_flags, const CTxUndo* txundo, TxVerbosity verbosity) { entry.pushKV("txid", tx.GetHash().GetHex()); entry.pushKV("hash", tx.GetWitnessHash().GetHex()); @@ -213,7 +216,7 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, if (verbosity == TxVerbosity::SHOW_DETAILS_AND_PREVOUT) { UniValue o_script_pub_key(UniValue::VOBJ); - ScriptPubKeyToUniv(prev_txout.scriptPubKey, o_script_pub_key, /*include_hex=*/ true); + ScriptToUniv(prev_txout.scriptPubKey, /*out=*/o_script_pub_key, /*include_hex=*/true, /*include_address=*/true); UniValue p(UniValue::VOBJ); p.pushKV("generated", bool(prev_coin.fCoinBase)); @@ -238,7 +241,7 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, out.pushKV("n", (int64_t)i); UniValue o(UniValue::VOBJ); - ScriptPubKeyToUniv(txout.scriptPubKey, o, true); + ScriptToUniv(txout.scriptPubKey, /*out=*/o, /*include_hex=*/true, /*include_address=*/true); out.pushKV("scriptPubKey", o); vout.push_back(out); @@ -254,8 +257,9 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, entry.pushKV("fee", ValueFromAmount(fee)); } - if (!hashBlock.IsNull()) - entry.pushKV("blockhash", hashBlock.GetHex()); + if (!block_hash.IsNull()) { + entry.pushKV("blockhash", block_hash.GetHex()); + } if (include_hex) { entry.pushKV("hex", EncodeHexTx(tx, serialize_flags)); // The hex-encoded transaction. Used the name "hex" to be consistent with the verbose output of "getrawtransaction". @@ -51,12 +51,26 @@ public: // Disallow std::string conversion method to avoid locale-dependent encoding on windows. std::string string() const = delete; + std::string u8string() const + { + const auto& utf8_str{std::filesystem::path::u8string()}; + // utf8_str might either be std::string (C++17) or std::u8string + // (C++20). Convert both to std::string. This method can be removed + // after switching to C++20. + return std::string{utf8_str.begin(), utf8_str.end()}; + } + // Required for path overloads in <fstream>. // See https://gcc.gnu.org/git/?p=gcc.git;a=commit;h=96e0367ead5d8dcac3bec2865582e76e2fbab190 path& make_preferred() { std::filesystem::path::make_preferred(); return *this; } path filename() const { return std::filesystem::path::filename(); } }; +static inline path u8path(const std::string& utf8_str) +{ + return std::filesystem::u8path(utf8_str); +} + // Disallow implicit std::string conversion for absolute to avoid // locale-dependent encoding on windows. static inline path absolute(const path& p) @@ -116,8 +130,8 @@ static inline std::string PathToString(const path& path) // use here, because these methods encode the path using C++'s narrow // multibyte encoding, which on Windows corresponds to the current "code // page", which is unpredictable and typically not able to represent all - // valid paths. So std::filesystem::path::u8string() and - // std::filesystem::u8path() functions are used instead on Windows. On + // valid paths. So fs::path::u8string() and + // fs::u8path() functions are used instead on Windows. On // POSIX, u8string/u8path functions are not safe to use because paths are // not always valid UTF-8, so plain string methods which do not transform // the path there are used. diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index 386eb67ce9..69078708f9 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -277,7 +277,7 @@ bool CoinStatsIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* n { LOCK(cs_main); - CBlockIndex* iter_tip{m_chainstate->m_blockman.LookupBlockIndex(current_tip->GetBlockHash())}; + const CBlockIndex* iter_tip{m_chainstate->m_blockman.LookupBlockIndex(current_tip->GetBlockHash())}; const auto& consensus_params{Params().GetConsensus()}; do { diff --git a/src/init.cpp b/src/init.cpp index a8b86453fe..83937c6925 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -444,7 +444,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-asmap=<file>", strprintf("Specify asn mapping used for bucketing of the peers (default: %s). Relative paths will be prefixed by the net-specific datadir location.", DEFAULT_ASMAP_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-bantime=<n>", strprintf("Default duration (in seconds) of manually configured bans (default: %u)", DEFAULT_MISBEHAVING_BANTIME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-bind=<addr>[:<port>][=onion]", strprintf("Bind to given address and always listen on it (default: 0.0.0.0). Use [host]:port notation for IPv6. Append =onion to tag any incoming connections to that address and port as incoming Tor connections (default: 127.0.0.1:%u=onion, testnet: 127.0.0.1:%u=onion, signet: 127.0.0.1:%u=onion, regtest: 127.0.0.1:%u=onion)", defaultBaseParams->OnionServiceTargetPort(), testnetBaseParams->OnionServiceTargetPort(), signetBaseParams->OnionServiceTargetPort(), regtestBaseParams->OnionServiceTargetPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); - argsman.AddArg("-cjdnsreachable", "If set then this host is configured for CJDNS (connecting to fc00::/8 addresses would lead us to the CJDNS network) (default: 0)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-cjdnsreachable", "If set, then this host is configured for CJDNS (connecting to fc00::/8 addresses would lead us to the CJDNS network, see doc/cjdns.md) (default: 0)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-connect=<ip>", "Connect only to the specified node; -noconnect disables automatic connections (the rules for this peer are the same as for -addnode). This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-discover", "Discover own IP addresses (default: 1 when listening and no -externalip or -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-dns", strprintf("Allow DNS lookups for -addnode, -seednode and -connect (default: %u)", DEFAULT_NAME_LOOKUP), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -457,7 +457,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-maxconnections=<n>", strprintf("Maintain at most <n> connections to peers (default: %u). This limit does not apply to connections manually added via -addnode or the addnode RPC, which have a separate limit of %u.", DEFAULT_MAX_PEER_CONNECTIONS, MAX_ADDNODE_CONNECTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxreceivebuffer=<n>", strprintf("Maximum per-connection receive buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXRECEIVEBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxsendbuffer=<n>", strprintf("Maximum per-connection send buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXSENDBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-maxtimeadjustment", strprintf("Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by peers forward or backward by this amount. (default: %u seconds)", DEFAULT_MAX_TIME_ADJUSTMENT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-maxtimeadjustment", strprintf("Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by outbound peers forward or backward by this amount (default: %u seconds).", DEFAULT_MAX_TIME_ADJUSTMENT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxuploadtarget=<n>", strprintf("Tries to keep outbound traffic under the given target per 24h. Limit does not apply to peers with 'download' permission or blocks created within past week. 0 = no limit (default: %s). Optional suffix units [k|K|m|M|g|G|t|T] (default: M). Lowercase is 1000 base while uppercase is 1024 base", DEFAULT_MAX_UPLOAD_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-onion=<ip:port>", "Use separate SOCKS5 proxy to reach peers via Tor onion services, set -noonion to disable (default: -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-i2psam=<ip:port>", "I2P SAM proxy to reach I2P peers and accept I2P connections (default: none)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -853,12 +853,14 @@ bool AppInitParameterInteraction(const ArgsManager& args) nLocalServices = ServiceFlags(nLocalServices | NODE_COMPACT_FILTERS); } - // if using block pruning, then disallow txindex and coinstatsindex if (args.GetIntArg("-prune", 0)) { if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) return InitError(_("Prune mode is incompatible with -txindex.")); if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) return InitError(_("Prune mode is incompatible with -coinstatsindex.")); + if (args.GetBoolArg("-reindex-chainstate", false)) { + return InitError(_("Prune mode is incompatible with -reindex-chainstate. Use full -reindex instead.")); + } } // If -forcednsseed is set to true, ensure -dnsseed has not been set to false @@ -1300,6 +1302,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } for (int n = 0; n < NET_MAX; n++) { enum Network net = (enum Network)n; + assert(IsReachable(net)); if (!nets.count(net)) SetReachable(net, false); } @@ -1539,7 +1542,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // ********************************************************* Step 8: start indexers if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { - if (const auto error{CheckLegacyTxindex(*Assert(chainman.m_blockman.m_block_tree_db))}) { + if (const auto error{WITH_LOCK(cs_main, return CheckLegacyTxindex(*Assert(chainman.m_blockman.m_block_tree_db)))}) { return InitError(*error); } diff --git a/src/net.cpp b/src/net.cpp index 955eec46e3..602d56ab98 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -626,12 +626,6 @@ void CNode::CopyStats(CNodeStats& stats) X(addr); X(addrBind); stats.m_network = ConnectedThroughNetwork(); - if (m_tx_relay != nullptr) { - LOCK(m_tx_relay->cs_filter); - stats.fRelayTxes = m_tx_relay->fRelayTxes; - } else { - stats.fRelayTxes = false; - } X(m_last_send); X(m_last_recv); X(m_last_tx_time); @@ -658,11 +652,6 @@ void CNode::CopyStats(CNodeStats& stats) X(nRecvBytes); } X(m_permissionFlags); - if (m_tx_relay != nullptr) { - stats.minFeeFilter = m_tx_relay->minFeeFilter; - } else { - stats.minFeeFilter = 0; - } X(m_last_ping_time); X(m_min_ping_time); @@ -913,7 +902,7 @@ static bool CompareNodeTXTime(const NodeEvictionCandidate &a, const NodeEviction { // There is a fall-through here because it is common for a node to have more than a few peers that have not yet relayed txn. if (a.m_last_tx_time != b.m_last_tx_time) return a.m_last_tx_time < b.m_last_tx_time; - if (a.fRelayTxes != b.fRelayTxes) return b.fRelayTxes; + if (a.m_relay_txs != b.m_relay_txs) return b.m_relay_txs; if (a.fBloomFilter != b.fBloomFilter) return a.fBloomFilter; return a.m_connected > b.m_connected; } @@ -921,7 +910,7 @@ static bool CompareNodeTXTime(const NodeEvictionCandidate &a, const NodeEviction // Pick out the potential block-relay only peers, and sort them by last block time. static bool CompareNodeBlockRelayOnlyTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) { - if (a.fRelayTxes != b.fRelayTxes) return a.fRelayTxes; + if (a.m_relay_txs != b.m_relay_txs) return a.m_relay_txs; if (a.m_last_block_time != b.m_last_block_time) return a.m_last_block_time < b.m_last_block_time; if (a.fRelevantServices != b.fRelevantServices) return b.fRelevantServices; return a.m_connected > b.m_connected; @@ -1046,7 +1035,7 @@ void ProtectEvictionCandidatesByRatio(std::vector<NodeEvictionCandidate>& evicti EraseLastKElements(vEvictionCandidates, CompareNodeTXTime, 4); // Protect up to 8 non-tx-relay peers that have sent us novel blocks. EraseLastKElements(vEvictionCandidates, CompareNodeBlockRelayOnlyTime, 8, - [](const NodeEvictionCandidate& n) { return !n.fRelayTxes && n.fRelevantServices; }); + [](const NodeEvictionCandidate& n) { return !n.m_relay_txs && n.fRelevantServices; }); // Protect 4 nodes that most recently sent us novel blocks. // An attacker cannot manipulate this metric without performing useful work. @@ -1112,18 +1101,11 @@ bool CConnman::AttemptToEvictConnection() continue; if (node->fDisconnect) continue; - bool peer_relay_txes = false; - bool peer_filter_not_null = false; - if (node->m_tx_relay != nullptr) { - LOCK(node->m_tx_relay->cs_filter); - peer_relay_txes = node->m_tx_relay->fRelayTxes; - peer_filter_not_null = node->m_tx_relay->pfilter != nullptr; - } NodeEvictionCandidate candidate = {node->GetId(), node->m_connected, node->m_min_ping_time, node->m_last_block_time, node->m_last_tx_time, HasAllDesirableServiceFlags(node->nServices), - peer_relay_txes, peer_filter_not_null, node->nKeyedNetGroup, - node->m_prefer_evict, node->addr.IsLocal(), + node->m_relays_txs.load(), node->m_bloom_filter_loaded.load(), + node->nKeyedNetGroup, node->m_prefer_evict, node->addr.IsLocal(), node->ConnectedThroughNetwork()}; vEvictionCandidates.push_back(candidate); } @@ -2800,7 +2782,7 @@ std::vector<CAddress> CConnman::GetAddresses(CNode& requestor, size_t max_addres auto r = m_addr_response_caches.emplace(cache_id, CachedAddrResponse{}); CachedAddrResponse& cache_entry = r.first->second; if (cache_entry.m_cache_entry_expiration < current_time) { // If emplace() added new one it has expiration 0. - cache_entry.m_addrs_response_cache = GetAddresses(max_addresses, max_pct, /* network */ std::nullopt); + cache_entry.m_addrs_response_cache = GetAddresses(max_addresses, max_pct, /*network=*/std::nullopt); // Choosing a proper cache lifetime is a trade-off between the privacy leak minimization // and the usefulness of ADDR responses to honest users. // @@ -3031,9 +3013,6 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, std::shared_ptr<Sock> s nLocalServices(nLocalServicesIn) { if (inbound_onion) assert(conn_type_in == ConnectionType::INBOUND); - if (conn_type_in != ConnectionType::BLOCK_RELAY) { - m_tx_relay = std::make_unique<TxRelay>(); - } for (const std::string &msg : getAllNetMessageTypes()) mapRecvBytesPerMsgCmd[msg] = 0; @@ -99,15 +99,22 @@ struct AddedNodeInfo class CNodeStats; class CClientUIInterface; -struct CSerializedNetMsg -{ +struct CSerializedNetMsg { CSerializedNetMsg() = default; CSerializedNetMsg(CSerializedNetMsg&&) = default; CSerializedNetMsg& operator=(CSerializedNetMsg&&) = default; - // No copying, only moves. + // No implicit copying, only moves. CSerializedNetMsg(const CSerializedNetMsg& msg) = delete; CSerializedNetMsg& operator=(const CSerializedNetMsg&) = delete; + CSerializedNetMsg Copy() const + { + CSerializedNetMsg copy; + copy.data = data; + copy.m_type = m_type; + return copy; + } + std::vector<unsigned char> data; std::string m_type; }; @@ -250,7 +257,6 @@ class CNodeStats public: NodeId nodeid; ServiceFlags nServices; - bool fRelayTxes; std::chrono::seconds m_last_send; std::chrono::seconds m_last_recv; std::chrono::seconds m_last_tx_time; @@ -271,7 +277,6 @@ public: NetPermissionFlags m_permissionFlags; std::chrono::microseconds m_last_ping_time; std::chrono::microseconds m_min_ping_time; - CAmount minFeeFilter; // Our address, as reported by the peer std::string addrLocal; // Address of this peer @@ -548,34 +553,15 @@ public: // Peer selected us as (compact blocks) high-bandwidth peer (BIP152) std::atomic<bool> m_bip152_highbandwidth_from{false}; - struct TxRelay { - mutable RecursiveMutex cs_filter; - // We use fRelayTxes for two purposes - - // a) it allows us to not relay tx invs before receiving the peer's version message - // b) the peer may tell us in its version message that we should not relay tx invs - // unless it loads a bloom filter. - bool fRelayTxes GUARDED_BY(cs_filter){false}; - std::unique_ptr<CBloomFilter> pfilter PT_GUARDED_BY(cs_filter) GUARDED_BY(cs_filter){nullptr}; - - mutable RecursiveMutex cs_tx_inventory; - CRollingBloomFilter filterInventoryKnown GUARDED_BY(cs_tx_inventory){50000, 0.000001}; - // Set of transaction ids we still have to announce. - // They are sorted by the mempool before relay, so the order is not important. - std::set<uint256> setInventoryTxToSend; - // Used for BIP35 mempool sending - bool fSendMempool GUARDED_BY(cs_tx_inventory){false}; - // Last time a "MEMPOOL" request was serviced. - std::atomic<std::chrono::seconds> m_last_mempool_req{0s}; - std::chrono::microseconds nNextInvSend{0}; - - /** Minimum fee rate with which to filter inv's to this node */ - std::atomic<CAmount> minFeeFilter{0}; - CAmount lastSentFeeFilter{0}; - std::chrono::microseconds m_next_send_feefilter{0}; - }; + /** Whether we should relay transactions to this peer (their version + * message did not include fRelay=false and this is not a block-relay-only + * connection). This only changes from false to true. It will never change + * back to false. Used only in inbound eviction logic. */ + std::atomic_bool m_relays_txs{false}; - // m_tx_relay == nullptr if we're not relaying transactions with this peer - std::unique_ptr<TxRelay> m_tx_relay; + /** Whether this peer has loaded a bloom filter. Used only in inbound + * eviction logic. */ + std::atomic_bool m_bloom_filter_loaded{false}; /** UNIX epoch time of the last block received from this peer that we had * not yet seen (e.g. not already received from another peer), that passed @@ -651,23 +637,6 @@ public: nRefCount--; } - void AddKnownTx(const uint256& hash) - { - if (m_tx_relay != nullptr) { - LOCK(m_tx_relay->cs_tx_inventory); - m_tx_relay->filterInventoryKnown.insert(hash); - } - } - - void PushTxInventory(const uint256& hash) - { - if (m_tx_relay == nullptr) return; - LOCK(m_tx_relay->cs_tx_inventory); - if (!m_tx_relay->filterInventoryKnown.contains(hash)) { - m_tx_relay->setInventoryTxToSend.insert(hash); - } - } - void CloseSocketDisconnect(); void CopyStats(CNodeStats& stats); @@ -1301,7 +1270,7 @@ struct NodeEvictionCandidate std::chrono::seconds m_last_block_time; std::chrono::seconds m_last_tx_time; bool fRelevantServices; - bool fRelayTxes; + bool m_relay_txs; bool fBloomFilter; uint64_t nKeyedNetGroup; bool prefer_evict; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 59cd83e493..ba72a11ec9 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -41,6 +41,7 @@ #include <algorithm> #include <atomic> #include <chrono> +#include <future> #include <memory> #include <optional> #include <typeinfo> @@ -232,6 +233,39 @@ struct Peer { /** Whether a ping has been requested by the user */ std::atomic<bool> m_ping_queued{false}; + /** Whether this peer relays txs via wtxid */ + std::atomic<bool> m_wtxid_relay{false}; + + struct TxRelay { + mutable RecursiveMutex m_bloom_filter_mutex; + // We use m_relay_txs for two purposes - + // a) it allows us to not relay tx invs before receiving the peer's version message + // b) the peer may tell us in its version message that we should not relay tx invs + // unless it loads a bloom filter. + bool m_relay_txs GUARDED_BY(m_bloom_filter_mutex){false}; + std::unique_ptr<CBloomFilter> m_bloom_filter PT_GUARDED_BY(m_bloom_filter_mutex) GUARDED_BY(m_bloom_filter_mutex){nullptr}; + + mutable RecursiveMutex m_tx_inventory_mutex; + CRollingBloomFilter m_tx_inventory_known_filter GUARDED_BY(m_tx_inventory_mutex){50000, 0.000001}; + // Set of transaction ids we still have to announce. + // They are sorted by the mempool before relay, so the order is not important. + std::set<uint256> m_tx_inventory_to_send; + // Used for BIP35 mempool sending + bool m_send_mempool GUARDED_BY(m_tx_inventory_mutex){false}; + // Last time a "MEMPOOL" request was serviced. + std::atomic<std::chrono::seconds> m_last_mempool_req{0s}; + std::chrono::microseconds m_next_inv_send_time{0}; + + /** Minimum fee rate with which to filter inv's to this node */ + std::atomic<CAmount> m_fee_filter_received{0}; + CAmount m_fee_filter_sent{0}; + std::chrono::microseconds m_next_send_feefilter{0}; + }; + + /** Transaction relay data. Will be a nullptr if we're not relaying + * transactions with this peer (e.g. if it's a block-relay-only peer) */ + std::unique_ptr<TxRelay> m_tx_relay; + /** A vector of addresses to send to the peer, limited to MAX_ADDR_TO_SEND. */ std::vector<CAddress> m_addrs_to_send; /** Probabilistic filter to track recent addr messages relayed with this @@ -290,8 +324,9 @@ struct Peer { /** Work queue of items requested by this peer **/ std::deque<CInv> m_getdata_requests GUARDED_BY(m_getdata_requests_mutex); - explicit Peer(NodeId id) + explicit Peer(NodeId id, bool tx_relay) : m_id(id) + , m_tx_relay(tx_relay ? std::make_unique<TxRelay>() : nullptr) {} }; @@ -331,9 +366,6 @@ public: const std::chrono::microseconds time_received, const std::atomic<bool>& interruptMsgProc) override; private: - void _RelayTransaction(const uint256& txid, const uint256& wtxid) - EXCLUSIVE_LOCKS_REQUIRED(cs_main); - /** Consider evicting an outbound peer based on the amount of time they've been behind our tip */ void ConsiderEviction(CNode& pto, std::chrono::seconds time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -394,7 +426,7 @@ private: EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** Send a version message to a peer */ - void PushNodeVersion(CNode& pnode); + void PushNodeVersion(CNode& pnode, const Peer& peer); /** Send a ping message every PING_INTERVAL or if requested via RPC. May * mark the peer to be disconnected if a ping has timed out. @@ -415,7 +447,7 @@ private: void RelayAddress(NodeId originator, const CAddress& addr, bool fReachable); /** Send `feefilter` message. */ - void MaybeSendFeefilter(CNode& node, std::chrono::microseconds current_time); + void MaybeSendFeefilter(CNode& node, Peer& peer, std::chrono::microseconds current_time); const CChainParams& m_chainparams; CConnman& m_connman; @@ -464,7 +496,7 @@ private: std::map<uint256, std::pair<NodeId, bool>> mapBlockSource GUARDED_BY(cs_main); /** Number of peers with wtxid relay. */ - int m_wtxid_relay_peers GUARDED_BY(cs_main) = 0; + std::atomic<int> m_wtxid_relay_peers{0}; /** Number of outbound peers with m_chain_sync.m_protect. */ int m_outbound_peers_with_protect_from_disconnect GUARDED_BY(cs_main) = 0; @@ -779,9 +811,6 @@ struct CNodeState { //! A rolling bloom filter of all announced tx CInvs to this peer. CRollingBloomFilter m_recently_announced_invs = CRollingBloomFilter{INVENTORY_MAX_RECENT_RELAY, 0.000001}; - //! Whether this peer relays txs via wtxid - bool m_wtxid_relay{false}; - CNodeState(bool is_inbound) : m_is_inbound(is_inbound) {} }; @@ -826,6 +855,14 @@ static void PushAddress(Peer& peer, const CAddress& addr, FastRandomContext& ins } } +static void AddKnownTx(Peer& peer, const uint256& hash) +{ + if (peer.m_tx_relay != nullptr) { + LOCK(peer.m_tx_relay->m_tx_inventory_mutex); + peer.m_tx_relay->m_tx_inventory_known_filter.insert(hash); + } +} + static void UpdatePreferredDownload(const CNode& node, CNodeState* state) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { nPreferredDownload -= state->fPreferredDownload; @@ -1121,7 +1158,7 @@ void PeerManagerImpl::FindNextBlocksToDownload(NodeId nodeid, unsigned int count } // namespace -void PeerManagerImpl::PushNodeVersion(CNode& pnode) +void PeerManagerImpl::PushNodeVersion(CNode& pnode, const Peer& peer) { // Note that pnode->GetLocalServices() is a reflection of the local // services we were offering when the CNode object was created for this @@ -1136,7 +1173,7 @@ void PeerManagerImpl::PushNodeVersion(CNode& pnode) CService addr_you = addr.IsRoutable() && !IsProxy(addr) && addr.IsAddrV1Compatible() ? addr : CService(); uint64_t your_services{addr.nServices}; - const bool tx_relay = !m_ignore_incoming_txs && pnode.m_tx_relay != nullptr && !pnode.IsFeelerConn(); + const bool tx_relay = !m_ignore_incoming_txs && peer.m_tx_relay != nullptr && !pnode.IsFeelerConn(); m_connman.PushMessage(&pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, my_services, nTime, your_services, addr_you, // Together the pre-version-31402 serialization of CAddress "addrYou" (without nTime) my_services, CService(), // Together the pre-version-31402 serialization of CAddress "addrMe" (without nTime) @@ -1193,13 +1230,13 @@ void PeerManagerImpl::InitializeNode(CNode *pnode) mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(pnode->IsInboundConn())); assert(m_txrequest.Count(nodeid) == 0); } + PeerRef peer = std::make_shared<Peer>(nodeid, /*tx_relay=*/ !pnode->IsBlockOnlyConn()); { - PeerRef peer = std::make_shared<Peer>(nodeid); LOCK(m_peer_mutex); - m_peer_map.emplace_hint(m_peer_map.end(), nodeid, std::move(peer)); + m_peer_map.emplace_hint(m_peer_map.end(), nodeid, peer); } if (!pnode->IsInboundConn()) { - PushNodeVersion(*pnode); + PushNodeVersion(*pnode, *peer); } } @@ -1211,8 +1248,7 @@ void PeerManagerImpl::ReattemptInitialBroadcast(CScheduler& scheduler) CTransactionRef tx = m_mempool.get(txid); if (tx != nullptr) { - LOCK(cs_main); - _RelayTransaction(txid, tx->GetWitnessHash()); + RelayTransaction(txid, tx->GetWitnessHash()); } else { m_mempool.RemoveUnbroadcastTx(txid, true); } @@ -1239,6 +1275,8 @@ void PeerManagerImpl::FinalizeNode(const CNode& node) PeerRef peer = RemovePeer(nodeid); assert(peer != nullptr); misbehavior = WITH_LOCK(peer->m_misbehavior_mutex, return peer->m_misbehavior_score); + m_wtxid_relay_peers -= peer->m_wtxid_relay; + assert(m_wtxid_relay_peers >= 0); } CNodeState *state = State(nodeid); assert(state != nullptr); @@ -1256,8 +1294,6 @@ void PeerManagerImpl::FinalizeNode(const CNode& node) assert(m_peers_downloading_from >= 0); m_outbound_peers_with_protect_from_disconnect -= state->m_chain_sync.m_protect; assert(m_outbound_peers_with_protect_from_disconnect >= 0); - m_wtxid_relay_peers -= state->m_wtxid_relay; - assert(m_wtxid_relay_peers >= 0); mapNodeState.erase(nodeid); @@ -1330,6 +1366,14 @@ bool PeerManagerImpl::GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) c ping_wait = GetTime<std::chrono::microseconds>() - peer->m_ping_start.load(); } + if (peer->m_tx_relay != nullptr) { + stats.m_relay_txs = WITH_LOCK(peer->m_tx_relay->m_bloom_filter_mutex, return peer->m_tx_relay->m_relay_txs); + stats.m_fee_filter_received = peer->m_tx_relay->m_fee_filter_received.load(); + } else { + stats.m_relay_txs = false; + stats.m_fee_filter_received = 0; + } + stats.m_ping_wait = ping_wait; stats.m_addr_processed = peer->m_addr_processed.load(); stats.m_addr_rate_limited = peer->m_addr_rate_limited.load(); @@ -1596,6 +1640,8 @@ void PeerManagerImpl::NewPoWValidBlock(const CBlockIndex *pindex, const std::sha bool fWitnessEnabled = DeploymentActiveAt(*pindex, m_chainparams.GetConsensus(), Consensus::DEPLOYMENT_SEGWIT); uint256 hashBlock(pblock->GetHash()); + const std::shared_future<CSerializedNetMsg> lazy_ser{ + std::async(std::launch::deferred, [&] { return msgMaker.Make(NetMsgType::CMPCTBLOCK, *pcmpctblock); })}; { LOCK(cs_most_recent_block); @@ -1605,10 +1651,9 @@ void PeerManagerImpl::NewPoWValidBlock(const CBlockIndex *pindex, const std::sha fWitnessesPresentInMostRecentCompactBlock = fWitnessEnabled; } - m_connman.ForEachNode([this, &pcmpctblock, pindex, &msgMaker, fWitnessEnabled, &hashBlock](CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { + m_connman.ForEachNode([this, pindex, fWitnessEnabled, &lazy_ser, &hashBlock](CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { AssertLockHeld(::cs_main); - // TODO: Avoid the repeated-serialization here if (pnode->GetCommonVersion() < INVALID_CB_NO_BAN_VERSION || pnode->fDisconnect) return; ProcessBlockAvailability(pnode->GetId()); @@ -1620,7 +1665,9 @@ void PeerManagerImpl::NewPoWValidBlock(const CBlockIndex *pindex, const std::sha LogPrint(BCLog::NET, "%s sending header-and-ids %s to peer=%d\n", "PeerManager::NewPoWValidBlock", hashBlock.ToString(), pnode->GetId()); - m_connman.PushMessage(pnode, msgMaker.Make(NetMsgType::CMPCTBLOCK, *pcmpctblock)); + + const CSerializedNetMsg& ser_cmpctblock{lazy_ser.get()}; + m_connman.PushMessage(pnode, ser_cmpctblock.Copy()); state.pindexBestHeaderSent = pindex; } }); @@ -1742,22 +1789,17 @@ void PeerManagerImpl::SendPings() void PeerManagerImpl::RelayTransaction(const uint256& txid, const uint256& wtxid) { - WITH_LOCK(cs_main, _RelayTransaction(txid, wtxid);); -} - -void PeerManagerImpl::_RelayTransaction(const uint256& txid, const uint256& wtxid) -{ - m_connman.ForEachNode([&txid, &wtxid](CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { - AssertLockHeld(::cs_main); + LOCK(m_peer_mutex); + for(auto& it : m_peer_map) { + Peer& peer = *it.second; + if (!peer.m_tx_relay) continue; - CNodeState* state = State(pnode->GetId()); - if (state == nullptr) return; - if (state->m_wtxid_relay) { - pnode->PushTxInventory(wtxid); - } else { - pnode->PushTxInventory(txid); + const uint256& hash{peer.m_wtxid_relay ? wtxid : txid}; + LOCK(peer.m_tx_relay->m_tx_inventory_mutex); + if (!peer.m_tx_relay->m_tx_inventory_known_filter.contains(hash)) { + peer.m_tx_relay->m_tx_inventory_to_send.insert(hash); } - }); + }; } void PeerManagerImpl::RelayAddress(NodeId originator, @@ -1903,11 +1945,11 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& } else if (inv.IsMsgFilteredBlk()) { bool sendMerkleBlock = false; CMerkleBlock merkleBlock; - if (pfrom.m_tx_relay != nullptr) { - LOCK(pfrom.m_tx_relay->cs_filter); - if (pfrom.m_tx_relay->pfilter) { + if (peer.m_tx_relay != nullptr) { + LOCK(peer.m_tx_relay->m_bloom_filter_mutex); + if (peer.m_tx_relay->m_bloom_filter) { sendMerkleBlock = true; - merkleBlock = CMerkleBlock(*pblock, *pfrom.m_tx_relay->pfilter); + merkleBlock = CMerkleBlock(*pblock, *peer.m_tx_relay->m_bloom_filter); } } if (sendMerkleBlock) { @@ -1996,7 +2038,7 @@ void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic const auto now{GetTime<std::chrono::seconds>()}; // Get last mempool request time - const auto mempool_req = pfrom.m_tx_relay != nullptr ? pfrom.m_tx_relay->m_last_mempool_req.load() : std::chrono::seconds::min(); + const auto mempool_req = peer.m_tx_relay != nullptr ? peer.m_tx_relay->m_last_mempool_req.load() : std::chrono::seconds::min(); // Process as many TX items from the front of the getdata queue as // possible, since they're common and it's efficient to batch process @@ -2009,7 +2051,7 @@ void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic const CInv &inv = *it++; - if (pfrom.m_tx_relay == nullptr) { + if (peer.m_tx_relay == nullptr) { // Ignore GETDATA requests for transactions from blocks-only peers. continue; } @@ -2037,7 +2079,7 @@ void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic } for (const uint256& parent_txid : parent_ids_to_add) { // Relaying a transaction with a recent but unconfirmed parent. - if (WITH_LOCK(pfrom.m_tx_relay->cs_tx_inventory, return !pfrom.m_tx_relay->filterInventoryKnown.contains(parent_txid))) { + if (WITH_LOCK(peer.m_tx_relay->m_tx_inventory_mutex, return !peer.m_tx_relay->m_tx_inventory_known_filter.contains(parent_txid))) { LOCK(cs_main); State(pfrom.GetId())->m_recently_announced_invs.insert(parent_txid); } @@ -2317,7 +2359,7 @@ void PeerManagerImpl::ProcessOrphanTx(std::set<uint256>& orphan_work_set) if (result.m_result_type == MempoolAcceptResult::ResultType::VALID) { LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); - _RelayTransaction(orphanHash, porphanTx->GetWitnessHash()); + RelayTransaction(orphanHash, porphanTx->GetWitnessHash()); m_orphanage.AddChildrenToWorkSet(*porphanTx, orphan_work_set); m_orphanage.EraseTx(orphanHash); for (const CTransactionRef& removedTx : result.m_replaced_transactions.value()) { @@ -2633,7 +2675,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Inbound peers send us their version message when they connect. // We send our version message in response. if (pfrom.IsInboundConn()) { - PushNodeVersion(pfrom); + PushNodeVersion(pfrom, *peer); } // Change version @@ -2672,9 +2714,12 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // set nodes not capable of serving the complete blockchain history as "limited nodes" pfrom.m_limited_node = (!(nServices & NODE_NETWORK) && (nServices & NODE_NETWORK_LIMITED)); - if (pfrom.m_tx_relay != nullptr) { - LOCK(pfrom.m_tx_relay->cs_filter); - pfrom.m_tx_relay->fRelayTxes = fRelay; // set to true after we get the first filter* message + if (peer->m_tx_relay != nullptr) { + { + LOCK(peer->m_tx_relay->m_bloom_filter_mutex); + peer->m_tx_relay->m_relay_txs = fRelay; // set to true after we get the first filter* message + } + if (fRelay) pfrom.m_relays_txs = true; } if((nServices & NODE_WITNESS)) @@ -2863,9 +2908,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } if (pfrom.GetCommonVersion() >= WTXID_RELAY_VERSION) { - LOCK(cs_main); - if (!State(pfrom.GetId())->m_wtxid_relay) { - State(pfrom.GetId())->m_wtxid_relay = true; + if (!peer->m_wtxid_relay) { + peer->m_wtxid_relay = true; m_wtxid_relay_peers++; } else { LogPrint(BCLog::NET, "ignoring duplicate wtxidrelay from peer=%d\n", pfrom.GetId()); @@ -3001,7 +3045,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Reject tx INVs when the -blocksonly setting is enabled, or this is a // block-relay-only peer - bool reject_tx_invs{m_ignore_incoming_txs || (pfrom.m_tx_relay == nullptr)}; + bool reject_tx_invs{m_ignore_incoming_txs || (peer->m_tx_relay == nullptr)}; // Allow peers with relay permission to send data other than blocks in blocks only mode if (pfrom.HasPermission(NetPermissionFlags::Relay)) { @@ -3019,7 +3063,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Ignore INVs that don't match wtxidrelay setting. // Note that orphan parent fetching always uses MSG_TX GETDATAs regardless of the wtxidrelay setting. // This is fine as no INV messages are involved in that process. - if (State(pfrom.GetId())->m_wtxid_relay) { + if (peer->m_wtxid_relay) { if (inv.IsMsgTx()) continue; } else { if (inv.IsMsgWtx()) continue; @@ -3048,7 +3092,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, const bool fAlreadyHave = AlreadyHaveTx(gtxid); LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom.GetId()); - pfrom.AddKnownTx(inv.hash); + AddKnownTx(*peer, inv.hash); if (!fAlreadyHave && !m_chainman.ActiveChainstate().IsInitialBlockDownload()) { AddTxAnnouncement(pfrom, gtxid, current_time); } @@ -3278,8 +3322,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Stop processing the transaction early if // 1) We are in blocks only mode and peer has no relay permission // 2) This peer is a block-relay-only peer - if ((m_ignore_incoming_txs && !pfrom.HasPermission(NetPermissionFlags::Relay)) || (pfrom.m_tx_relay == nullptr)) - { + if ((m_ignore_incoming_txs && !pfrom.HasPermission(NetPermissionFlags::Relay)) || (peer->m_tx_relay == nullptr)) { LogPrint(BCLog::NET, "transaction sent in violation of protocol peer=%d\n", pfrom.GetId()); pfrom.fDisconnect = true; return; @@ -3297,21 +3340,19 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, const uint256& txid = ptx->GetHash(); const uint256& wtxid = ptx->GetWitnessHash(); - LOCK2(cs_main, g_cs_orphans); - - CNodeState* nodestate = State(pfrom.GetId()); - - const uint256& hash = nodestate->m_wtxid_relay ? wtxid : txid; - pfrom.AddKnownTx(hash); - if (nodestate->m_wtxid_relay && txid != wtxid) { - // Insert txid into filterInventoryKnown, even for + const uint256& hash = peer->m_wtxid_relay ? wtxid : txid; + AddKnownTx(*peer, hash); + if (peer->m_wtxid_relay && txid != wtxid) { + // Insert txid into m_tx_inventory_known_filter, even for // wtxidrelay peers. This prevents re-adding of // unconfirmed parents to the recently_announced // filter, when a child tx is requested. See // ProcessGetData(). - pfrom.AddKnownTx(txid); + AddKnownTx(*peer, txid); } + LOCK2(cs_main, g_cs_orphans); + m_txrequest.ReceivedResponse(pfrom.GetId(), txid); if (tx.HasWitness()) m_txrequest.ReceivedResponse(pfrom.GetId(), wtxid); @@ -3336,7 +3377,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, LogPrintf("Not relaying non-mempool transaction %s from forcerelay peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); } else { LogPrintf("Force relaying tx %s from peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); - _RelayTransaction(tx.GetHash(), tx.GetWitnessHash()); + RelayTransaction(tx.GetHash(), tx.GetWitnessHash()); } } return; @@ -3350,7 +3391,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // requests for it. m_txrequest.ForgetTxHash(tx.GetHash()); m_txrequest.ForgetTxHash(tx.GetWitnessHash()); - _RelayTransaction(tx.GetHash(), tx.GetWitnessHash()); + RelayTransaction(tx.GetHash(), tx.GetWitnessHash()); m_orphanage.AddChildrenToWorkSet(tx, peer->m_orphan_work_set); pfrom.m_last_tx_time = GetTime<std::chrono::seconds>(); @@ -3397,7 +3438,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Eventually we should replace this with an improved // protocol for getting all unconfirmed parents. const auto gtxid{GenTxid::Txid(parent_txid)}; - pfrom.AddKnownTx(parent_txid); + AddKnownTx(*peer, parent_txid); if (!AlreadyHaveTx(gtxid)) AddTxAnnouncement(pfrom, gtxid, current_time); } @@ -3521,7 +3562,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, BlockValidationState state; if (!m_chainman.ProcessNewBlockHeaders({cmpctblock.header}, state, m_chainparams, &pindex)) { if (state.IsInvalid()) { - MaybePunishNodeForBlock(pfrom.GetId(), state, /*via_compact_block*/ true, "invalid header via cmpctblock"); + MaybePunishNodeForBlock(pfrom.GetId(), state, /*via_compact_block=*/true, "invalid header via cmpctblock"); return; } } @@ -3861,7 +3902,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, peer->m_addrs_to_send.clear(); std::vector<CAddress> vAddr; if (pfrom.HasPermission(NetPermissionFlags::Addr)) { - vAddr = m_connman.GetAddresses(MAX_ADDR_TO_SEND, MAX_PCT_ADDR_TO_SEND, /* network */ std::nullopt); + vAddr = m_connman.GetAddresses(MAX_ADDR_TO_SEND, MAX_PCT_ADDR_TO_SEND, /*network=*/std::nullopt); } else { vAddr = m_connman.GetAddresses(pfrom, MAX_ADDR_TO_SEND, MAX_PCT_ADDR_TO_SEND); } @@ -3893,9 +3934,9 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } - if (pfrom.m_tx_relay != nullptr) { - LOCK(pfrom.m_tx_relay->cs_tx_inventory); - pfrom.m_tx_relay->fSendMempool = true; + if (peer->m_tx_relay != nullptr) { + LOCK(peer->m_tx_relay->m_tx_inventory_mutex); + peer->m_tx_relay->m_send_mempool = true; } return; } @@ -3989,11 +4030,15 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // There is no excuse for sending a too-large filter Misbehaving(pfrom.GetId(), 100, "too-large bloom filter"); } - else if (pfrom.m_tx_relay != nullptr) + else if (peer->m_tx_relay != nullptr) { - LOCK(pfrom.m_tx_relay->cs_filter); - pfrom.m_tx_relay->pfilter.reset(new CBloomFilter(filter)); - pfrom.m_tx_relay->fRelayTxes = true; + { + LOCK(peer->m_tx_relay->m_bloom_filter_mutex); + peer->m_tx_relay->m_bloom_filter.reset(new CBloomFilter(filter)); + peer->m_tx_relay->m_relay_txs = true; + } + pfrom.m_bloom_filter_loaded = true; + pfrom.m_relays_txs = true; } return; } @@ -4012,10 +4057,10 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, bool bad = false; if (vData.size() > MAX_SCRIPT_ELEMENT_SIZE) { bad = true; - } else if (pfrom.m_tx_relay != nullptr) { - LOCK(pfrom.m_tx_relay->cs_filter); - if (pfrom.m_tx_relay->pfilter) { - pfrom.m_tx_relay->pfilter->insert(vData); + } else if (peer->m_tx_relay != nullptr) { + LOCK(peer->m_tx_relay->m_bloom_filter_mutex); + if (peer->m_tx_relay->m_bloom_filter) { + peer->m_tx_relay->m_bloom_filter->insert(vData); } else { bad = true; } @@ -4032,12 +4077,17 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, pfrom.fDisconnect = true; return; } - if (pfrom.m_tx_relay == nullptr) { + if (peer->m_tx_relay == nullptr) { return; } - LOCK(pfrom.m_tx_relay->cs_filter); - pfrom.m_tx_relay->pfilter = nullptr; - pfrom.m_tx_relay->fRelayTxes = true; + + { + LOCK(peer->m_tx_relay->m_bloom_filter_mutex); + peer->m_tx_relay->m_bloom_filter = nullptr; + peer->m_tx_relay->m_relay_txs = true; + } + pfrom.m_bloom_filter_loaded = false; + pfrom.m_relays_txs = true; return; } @@ -4045,8 +4095,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, CAmount newFeeFilter = 0; vRecv >> newFeeFilter; if (MoneyRange(newFeeFilter)) { - if (pfrom.m_tx_relay != nullptr) { - pfrom.m_tx_relay->minFeeFilter = newFeeFilter; + if (peer->m_tx_relay != nullptr) { + peer->m_tx_relay->m_fee_filter_received = newFeeFilter; } LogPrint(BCLog::NET, "received: feefilter of %s from peer=%d\n", CFeeRate(newFeeFilter).ToString(), pfrom.GetId()); } @@ -4504,10 +4554,10 @@ void PeerManagerImpl::MaybeSendAddr(CNode& node, Peer& peer, std::chrono::micros } } -void PeerManagerImpl::MaybeSendFeefilter(CNode& pto, std::chrono::microseconds current_time) +void PeerManagerImpl::MaybeSendFeefilter(CNode& pto, Peer& peer, std::chrono::microseconds current_time) { if (m_ignore_incoming_txs) return; - if (!pto.m_tx_relay) return; + if (!peer.m_tx_relay) return; if (pto.GetCommonVersion() < FEEFILTER_VERSION) return; // peers with the forcerelay permission should not filter txs to us if (pto.HasPermission(NetPermissionFlags::ForceRelay)) return; @@ -4521,27 +4571,27 @@ void PeerManagerImpl::MaybeSendFeefilter(CNode& pto, std::chrono::microseconds c currentFilter = MAX_MONEY; } else { static const CAmount MAX_FILTER{g_filter_rounder.round(MAX_MONEY)}; - if (pto.m_tx_relay->lastSentFeeFilter == MAX_FILTER) { + if (peer.m_tx_relay->m_fee_filter_sent == MAX_FILTER) { // Send the current filter if we sent MAX_FILTER previously // and made it out of IBD. - pto.m_tx_relay->m_next_send_feefilter = 0us; + peer.m_tx_relay->m_next_send_feefilter = 0us; } } - if (current_time > pto.m_tx_relay->m_next_send_feefilter) { + if (current_time > peer.m_tx_relay->m_next_send_feefilter) { CAmount filterToSend = g_filter_rounder.round(currentFilter); // We always have a fee filter of at least minRelayTxFee filterToSend = std::max(filterToSend, ::minRelayTxFee.GetFeePerK()); - if (filterToSend != pto.m_tx_relay->lastSentFeeFilter) { + if (filterToSend != peer.m_tx_relay->m_fee_filter_sent) { m_connman.PushMessage(&pto, CNetMsgMaker(pto.GetCommonVersion()).Make(NetMsgType::FEEFILTER, filterToSend)); - pto.m_tx_relay->lastSentFeeFilter = filterToSend; + peer.m_tx_relay->m_fee_filter_sent = filterToSend; } - pto.m_tx_relay->m_next_send_feefilter = GetExponentialRand(current_time, AVG_FEEFILTER_BROADCAST_INTERVAL); + peer.m_tx_relay->m_next_send_feefilter = GetExponentialRand(current_time, AVG_FEEFILTER_BROADCAST_INTERVAL); } // If the fee filter has changed substantially and it's still more than MAX_FEEFILTER_CHANGE_DELAY // until scheduled broadcast, then move the broadcast to within MAX_FEEFILTER_CHANGE_DELAY. - else if (current_time + MAX_FEEFILTER_CHANGE_DELAY < pto.m_tx_relay->m_next_send_feefilter && - (currentFilter < 3 * pto.m_tx_relay->lastSentFeeFilter / 4 || currentFilter > 4 * pto.m_tx_relay->lastSentFeeFilter / 3)) { - pto.m_tx_relay->m_next_send_feefilter = current_time + GetRandomDuration<std::chrono::microseconds>(MAX_FEEFILTER_CHANGE_DELAY); + else if (current_time + MAX_FEEFILTER_CHANGE_DELAY < peer.m_tx_relay->m_next_send_feefilter && + (currentFilter < 3 * peer.m_tx_relay->m_fee_filter_sent / 4 || currentFilter > 4 * peer.m_tx_relay->m_fee_filter_sent / 3)) { + peer.m_tx_relay->m_next_send_feefilter = current_time + GetRandomDuration<std::chrono::microseconds>(MAX_FEEFILTER_CHANGE_DELAY); } } @@ -4808,45 +4858,45 @@ bool PeerManagerImpl::SendMessages(CNode* pto) peer->m_blocks_for_inv_relay.clear(); } - if (pto->m_tx_relay != nullptr) { - LOCK(pto->m_tx_relay->cs_tx_inventory); + if (peer->m_tx_relay != nullptr) { + LOCK(peer->m_tx_relay->m_tx_inventory_mutex); // Check whether periodic sends should happen bool fSendTrickle = pto->HasPermission(NetPermissionFlags::NoBan); - if (pto->m_tx_relay->nNextInvSend < current_time) { + if (peer->m_tx_relay->m_next_inv_send_time < current_time) { fSendTrickle = true; if (pto->IsInboundConn()) { - pto->m_tx_relay->nNextInvSend = NextInvToInbounds(current_time, INBOUND_INVENTORY_BROADCAST_INTERVAL); + peer->m_tx_relay->m_next_inv_send_time = NextInvToInbounds(current_time, INBOUND_INVENTORY_BROADCAST_INTERVAL); } else { - pto->m_tx_relay->nNextInvSend = GetExponentialRand(current_time, OUTBOUND_INVENTORY_BROADCAST_INTERVAL); + peer->m_tx_relay->m_next_inv_send_time = GetExponentialRand(current_time, OUTBOUND_INVENTORY_BROADCAST_INTERVAL); } } // Time to send but the peer has requested we not relay transactions. if (fSendTrickle) { - LOCK(pto->m_tx_relay->cs_filter); - if (!pto->m_tx_relay->fRelayTxes) pto->m_tx_relay->setInventoryTxToSend.clear(); + LOCK(peer->m_tx_relay->m_bloom_filter_mutex); + if (!peer->m_tx_relay->m_relay_txs) peer->m_tx_relay->m_tx_inventory_to_send.clear(); } // Respond to BIP35 mempool requests - if (fSendTrickle && pto->m_tx_relay->fSendMempool) { + if (fSendTrickle && peer->m_tx_relay->m_send_mempool) { auto vtxinfo = m_mempool.infoAll(); - pto->m_tx_relay->fSendMempool = false; - const CFeeRate filterrate{pto->m_tx_relay->minFeeFilter.load()}; + peer->m_tx_relay->m_send_mempool = false; + const CFeeRate filterrate{peer->m_tx_relay->m_fee_filter_received.load()}; - LOCK(pto->m_tx_relay->cs_filter); + LOCK(peer->m_tx_relay->m_bloom_filter_mutex); for (const auto& txinfo : vtxinfo) { - const uint256& hash = state.m_wtxid_relay ? txinfo.tx->GetWitnessHash() : txinfo.tx->GetHash(); - CInv inv(state.m_wtxid_relay ? MSG_WTX : MSG_TX, hash); - pto->m_tx_relay->setInventoryTxToSend.erase(hash); + const uint256& hash = peer->m_wtxid_relay ? txinfo.tx->GetWitnessHash() : txinfo.tx->GetHash(); + CInv inv(peer->m_wtxid_relay ? MSG_WTX : MSG_TX, hash); + peer->m_tx_relay->m_tx_inventory_to_send.erase(hash); // Don't send transactions that peers will not put into their mempool if (txinfo.fee < filterrate.GetFee(txinfo.vsize)) { continue; } - if (pto->m_tx_relay->pfilter) { - if (!pto->m_tx_relay->pfilter->IsRelevantAndUpdate(*txinfo.tx)) continue; + if (peer->m_tx_relay->m_bloom_filter) { + if (!peer->m_tx_relay->m_bloom_filter->IsRelevantAndUpdate(*txinfo.tx)) continue; } - pto->m_tx_relay->filterInventoryKnown.insert(hash); + peer->m_tx_relay->m_tx_inventory_known_filter.insert(hash); // Responses to MEMPOOL requests bypass the m_recently_announced_invs filter. vInv.push_back(inv); if (vInv.size() == MAX_INV_SZ) { @@ -4854,37 +4904,37 @@ bool PeerManagerImpl::SendMessages(CNode* pto) vInv.clear(); } } - pto->m_tx_relay->m_last_mempool_req = std::chrono::duration_cast<std::chrono::seconds>(current_time); + peer->m_tx_relay->m_last_mempool_req = std::chrono::duration_cast<std::chrono::seconds>(current_time); } // Determine transactions to relay if (fSendTrickle) { // Produce a vector with all candidates for sending std::vector<std::set<uint256>::iterator> vInvTx; - vInvTx.reserve(pto->m_tx_relay->setInventoryTxToSend.size()); - for (std::set<uint256>::iterator it = pto->m_tx_relay->setInventoryTxToSend.begin(); it != pto->m_tx_relay->setInventoryTxToSend.end(); it++) { + vInvTx.reserve(peer->m_tx_relay->m_tx_inventory_to_send.size()); + for (std::set<uint256>::iterator it = peer->m_tx_relay->m_tx_inventory_to_send.begin(); it != peer->m_tx_relay->m_tx_inventory_to_send.end(); it++) { vInvTx.push_back(it); } - const CFeeRate filterrate{pto->m_tx_relay->minFeeFilter.load()}; + const CFeeRate filterrate{peer->m_tx_relay->m_fee_filter_received.load()}; // Topologically and fee-rate sort the inventory we send for privacy and priority reasons. // A heap is used so that not all items need sorting if only a few are being sent. - CompareInvMempoolOrder compareInvMempoolOrder(&m_mempool, state.m_wtxid_relay); + CompareInvMempoolOrder compareInvMempoolOrder(&m_mempool, peer->m_wtxid_relay); std::make_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder); // No reason to drain out at many times the network's capacity, // especially since we have many peers and some will draw much shorter delays. unsigned int nRelayedTransactions = 0; - LOCK(pto->m_tx_relay->cs_filter); + LOCK(peer->m_tx_relay->m_bloom_filter_mutex); while (!vInvTx.empty() && nRelayedTransactions < INVENTORY_BROADCAST_MAX) { // Fetch the top element from the heap std::pop_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder); std::set<uint256>::iterator it = vInvTx.back(); vInvTx.pop_back(); uint256 hash = *it; - CInv inv(state.m_wtxid_relay ? MSG_WTX : MSG_TX, hash); + CInv inv(peer->m_wtxid_relay ? MSG_WTX : MSG_TX, hash); // Remove it from the to-be-sent set - pto->m_tx_relay->setInventoryTxToSend.erase(it); + peer->m_tx_relay->m_tx_inventory_to_send.erase(it); // Check if not in the filter already - if (pto->m_tx_relay->filterInventoryKnown.contains(hash)) { + if (peer->m_tx_relay->m_tx_inventory_known_filter.contains(hash)) { continue; } // Not in the mempool anymore? don't bother sending it. @@ -4898,7 +4948,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) if (txinfo.fee < filterrate.GetFee(txinfo.vsize)) { continue; } - if (pto->m_tx_relay->pfilter && !pto->m_tx_relay->pfilter->IsRelevantAndUpdate(*txinfo.tx)) continue; + if (peer->m_tx_relay->m_bloom_filter && !peer->m_tx_relay->m_bloom_filter->IsRelevantAndUpdate(*txinfo.tx)) continue; // Send State(pto->GetId())->m_recently_announced_invs.insert(hash); vInv.push_back(inv); @@ -4925,14 +4975,14 @@ bool PeerManagerImpl::SendMessages(CNode* pto) m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } - pto->m_tx_relay->filterInventoryKnown.insert(hash); + peer->m_tx_relay->m_tx_inventory_known_filter.insert(hash); if (hash != txid) { - // Insert txid into filterInventoryKnown, even for + // Insert txid into m_tx_inventory_known_filter, even for // wtxidrelay peers. This prevents re-adding of // unconfirmed parents to the recently_announced // filter, when a child tx is requested. See // ProcessGetData(). - pto->m_tx_relay->filterInventoryKnown.insert(txid); + peer->m_tx_relay->m_tx_inventory_known_filter.insert(txid); } } } @@ -5053,6 +5103,6 @@ bool PeerManagerImpl::SendMessages(CNode* pto) if (!vGetData.empty()) m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); } // release cs_main - MaybeSendFeefilter(*pto, current_time); + MaybeSendFeefilter(*pto, *peer, current_time); return true; } diff --git a/src/net_processing.h b/src/net_processing.h index e30f9f516c..7dacaee5c1 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -29,6 +29,8 @@ struct CNodeStateStats { int m_starting_height = -1; std::chrono::microseconds m_ping_wait; std::vector<int> vHeightInFlight; + bool m_relay_txs; + CAmount m_fee_filter_received; uint64_t m_addr_processed = 0; uint64_t m_addr_rate_limited = 0; bool m_addr_relay_enabled{false}; diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 7392830261..763fd29744 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -28,10 +28,45 @@ bool fHavePruned = false; bool fPruneMode = false; uint64_t nPruneTarget = 0; +bool CBlockIndexWorkComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const +{ + // First sort by most total work, ... + if (pa->nChainWork > pb->nChainWork) return false; + if (pa->nChainWork < pb->nChainWork) return true; + + // ... then by earliest time received, ... + if (pa->nSequenceId < pb->nSequenceId) return false; + if (pa->nSequenceId > pb->nSequenceId) return true; + + // Use pointer address as tie breaker (should only happen with blocks + // loaded from disk, as those all have id 0). + if (pa < pb) return false; + if (pa > pb) return true; + + // Identical blocks. + return false; +} + +bool CBlockIndexHeightOnlyComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const +{ + return pa->nHeight < pb->nHeight; +} + static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false); static FlatFileSeq BlockFileSeq(); static FlatFileSeq UndoFileSeq(); +std::vector<CBlockIndex*> BlockManager::GetAllBlockIndices() +{ + AssertLockHeld(cs_main); + std::vector<CBlockIndex*> rv; + rv.reserve(m_block_index.size()); + for (auto& [_, block_index] : m_block_index) { + rv.push_back(&block_index); + } + return rv; +} + CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) { AssertLockHeld(cs_main); @@ -203,8 +238,7 @@ CBlockIndex* BlockManager::InsertBlockIndex(const uint256& hash) return nullptr; } - // Return existing or create new - auto [mi, inserted] = m_block_index.try_emplace(hash); + const auto [mi, inserted]{m_block_index.try_emplace(hash)}; CBlockIndex* pindex = &(*mi).second; if (inserted) { pindex->phashBlock = &((*mi).first); @@ -212,46 +246,19 @@ CBlockIndex* BlockManager::InsertBlockIndex(const uint256& hash) return pindex; } -bool BlockManager::LoadBlockIndex( - const Consensus::Params& consensus_params, - ChainstateManager& chainman) +bool BlockManager::LoadBlockIndex(const Consensus::Params& consensus_params) { if (!m_block_tree_db->LoadBlockIndexGuts(consensus_params, [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); })) { return false; } // Calculate nChainWork - std::vector<std::pair<int, CBlockIndex*>> vSortedByHeight; - vSortedByHeight.reserve(m_block_index.size()); - for (auto& [_, block_index] : m_block_index) { - CBlockIndex* pindex = &block_index; - vSortedByHeight.push_back(std::make_pair(pindex->nHeight, pindex)); - } - sort(vSortedByHeight.begin(), vSortedByHeight.end()); - - // Find start of assumed-valid region. - int first_assumed_valid_height = std::numeric_limits<int>::max(); - - for (const auto& [height, block] : vSortedByHeight) { - if (block->IsAssumedValid()) { - auto chainstates = chainman.GetAll(); - - // If we encounter an assumed-valid block index entry, ensure that we have - // one chainstate that tolerates assumed-valid entries and another that does - // not (i.e. the background validation chainstate), since assumed-valid - // entries should always be pending validation by a fully-validated chainstate. - auto any_chain = [&](auto fnc) { return std::any_of(chainstates.cbegin(), chainstates.cend(), fnc); }; - assert(any_chain([](auto chainstate) { return chainstate->reliesOnAssumedValid(); })); - assert(any_chain([](auto chainstate) { return !chainstate->reliesOnAssumedValid(); })); - - first_assumed_valid_height = height; - break; - } - } + std::vector<CBlockIndex*> vSortedByHeight{GetAllBlockIndices()}; + std::sort(vSortedByHeight.begin(), vSortedByHeight.end(), + CBlockIndexHeightOnlyComparator()); - for (const std::pair<int, CBlockIndex*>& item : vSortedByHeight) { + for (CBlockIndex* pindex : vSortedByHeight) { if (ShutdownRequested()) return false; - CBlockIndex* pindex = item.second; pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex); pindex->nTimeMax = (pindex->pprev ? std::max(pindex->pprev->nTimeMax, pindex->nTime) : pindex->nTime); @@ -275,43 +282,6 @@ bool BlockManager::LoadBlockIndex( pindex->nStatus |= BLOCK_FAILED_CHILD; m_dirty_blockindex.insert(pindex); } - if (pindex->IsAssumedValid() || - (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && - (pindex->HaveTxsDownloaded() || pindex->pprev == nullptr))) { - - // Fill each chainstate's block candidate set. Only add assumed-valid - // blocks to the tip candidate set if the chainstate is allowed to rely on - // assumed-valid blocks. - // - // If all setBlockIndexCandidates contained the assumed-valid blocks, the - // background chainstate's ActivateBestChain() call would add assumed-valid - // blocks to the chain (based on how FindMostWorkChain() works). Obviously - // we don't want this since the purpose of the background validation chain - // is to validate assued-valid blocks. - // - // Note: This is considering all blocks whose height is greater or equal to - // the first assumed-valid block to be assumed-valid blocks, and excluding - // them from the background chainstate's setBlockIndexCandidates set. This - // does mean that some blocks which are not technically assumed-valid - // (later blocks on a fork beginning before the first assumed-valid block) - // might not get added to the background chainstate, but this is ok, - // because they will still be attached to the active chainstate if they - // actually contain more work. - // - // Instead of this height-based approach, an earlier attempt was made at - // detecting "holistically" whether the block index under consideration - // relied on an assumed-valid ancestor, but this proved to be too slow to - // be practical. - for (CChainState* chainstate : chainman.GetAll()) { - if (chainstate->reliesOnAssumedValid() || - pindex->nHeight < first_assumed_valid_height) { - chainstate->setBlockIndexCandidates.insert(pindex); - } - } - } - if (pindex->nStatus & BLOCK_FAILED_MASK && (!chainman.m_best_invalid || pindex->nChainWork > chainman.m_best_invalid->nChainWork)) { - chainman.m_best_invalid = pindex; - } if (pindex->pprev) { pindex->BuildSkip(); } @@ -355,9 +325,9 @@ bool BlockManager::WriteBlockIndexDB() return true; } -bool BlockManager::LoadBlockIndexDB(ChainstateManager& chainman) +bool BlockManager::LoadBlockIndexDB() { - if (!LoadBlockIndex(::Params().GetConsensus(), chainman)) { + if (!LoadBlockIndex(::Params().GetConsensus())) { return false; } @@ -382,9 +352,8 @@ bool BlockManager::LoadBlockIndexDB(ChainstateManager& chainman) LogPrintf("Checking all blk files are present...\n"); std::set<int> setBlkDataFiles; for (const auto& [_, block_index] : m_block_index) { - const CBlockIndex* pindex = &block_index; - if (pindex->nStatus & BLOCK_HAVE_DATA) { - setBlkDataFiles.insert(pindex->nFile); + if (block_index.nStatus & BLOCK_HAVE_DATA) { + setBlkDataFiles.insert(block_index.nFile); } } for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) { @@ -408,13 +377,13 @@ bool BlockManager::LoadBlockIndexDB(ChainstateManager& chainman) return true; } -CBlockIndex* BlockManager::GetLastCheckpoint(const CCheckpointData& data) +const CBlockIndex* BlockManager::GetLastCheckpoint(const CCheckpointData& data) { const MapCheckpoints& checkpoints = data.mapCheckpoints; for (const MapCheckpoints::value_type& i : reverse_iterate(checkpoints)) { const uint256& hash = i.second; - CBlockIndex* pindex = LookupBlockIndex(hash); + const CBlockIndex* pindex = LookupBlockIndex(hash); if (pindex) { return pindex; } diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 12224f7a5d..a051e90808 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -62,6 +62,11 @@ struct CBlockIndexWorkComparator { bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const; }; +struct CBlockIndexHeightOnlyComparator { + /* Only compares the height of two block indices, doesn't try to tie-break */ + bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const; +}; + /** * Maintains a tree of blocks (stored in `m_block_index`) which is consulted * to determine where the most-work tip is. @@ -118,6 +123,8 @@ private: public: BlockMap m_block_index GUARDED_BY(cs_main); + std::vector<CBlockIndex*> GetAllBlockIndices() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + /** * All pairs A->B, where A (or one of its ancestors) misses transactions, but B has transactions. * Pruned nodes may have entries where B is missing data. @@ -127,16 +134,15 @@ public: std::unique_ptr<CBlockTreeDB> m_block_tree_db GUARDED_BY(::cs_main); bool WriteBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - bool LoadBlockIndexDB(ChainstateManager& chainman) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + bool LoadBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** * Load the blocktree off disk and into memory. Populate certain metadata * per index entry (nStatus, nChainWork, nTimeMax, etc.) as well as peripheral * collections like m_dirty_blockindex. */ - bool LoadBlockIndex( - const Consensus::Params& consensus_params, - ChainstateManager& chainman) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool LoadBlockIndex(const Consensus::Params& consensus_params) + EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Clear all data members. */ void Unload() EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -163,7 +169,7 @@ public: uint64_t CalculateCurrentUsage(); //! Returns last CBlockIndex* that is a checkpoint - CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + const CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOCKS_REQUIRED(cs_main); ~BlockManager() { diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index cb063ae9f8..d71455bc37 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -490,7 +490,7 @@ public: { LOCK(cs_main); const CChainState& active = Assert(m_node.chainman)->ActiveChainstate(); - if (CBlockIndex* fork = active.FindForkInGlobalIndex(locator)) { + if (const CBlockIndex* fork = active.FindForkInGlobalIndex(locator)) { return fork->nHeight; } return std::nullopt; @@ -557,7 +557,7 @@ public: // used to limit the range, and passing min_height that's too low or // max_height that's too high will not crash or change the result. LOCK(::cs_main); - if (CBlockIndex* block = chainman().m_blockman.LookupBlockIndex(block_hash)) { + if (const CBlockIndex* block = chainman().m_blockman.LookupBlockIndex(block_hash)) { if (max_height && block->nHeight >= *max_height) block = block->GetAncestor(*max_height); for (; block->nStatus & BLOCK_HAVE_DATA; block = block->pprev) { // Check pprev to not segfault if min_height is too low @@ -590,7 +590,7 @@ public: bool relay, std::string& err_string) override { - const TransactionError err = BroadcastTransaction(m_node, tx, err_string, max_tx_fee, relay, /*wait_callback*/ false); + const TransactionError err = BroadcastTransaction(m_node, tx, err_string, max_tx_fee, relay, /*wait_callback=*/false); // Chain clients only care about failures to accept the tx to the mempool. Disregard non-mempool related failures. // Note: this will need to be updated if BroadcastTransactions() is updated to return other non-mempool failures // that Chain clients do not need to know about. diff --git a/src/node/miner.cpp b/src/node/miner.cpp index 7fe10ecabc..917df91933 100644 --- a/src/node/miner.cpp +++ b/src/node/miner.cpp @@ -50,7 +50,7 @@ void RegenerateCommitments(CBlock& block, ChainstateManager& chainman) tx.vout.erase(tx.vout.begin() + GetWitnessCommitmentIndex(block)); block.vtx.at(0) = MakeTransactionRef(tx); - CBlockIndex* prev_block = WITH_LOCK(::cs_main, return chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock)); + const CBlockIndex* prev_block = WITH_LOCK(::cs_main, return chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock)); GenerateCoinbaseCommitment(block, prev_block, Params().GetConsensus()); block.hashMerkleRoot = BlockMerkleRoot(block); @@ -97,7 +97,6 @@ void BlockAssembler::resetBlock() // Reserve space for coinbase tx nBlockWeight = 4000; nBlockSigOpsCost = 400; - fIncludeWitness = false; // These counters do not include coinbase tx nBlockTx = 0; @@ -137,17 +136,6 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc pblock->nTime = GetAdjustedTime(); m_lock_time_cutoff = pindexPrev->GetMedianTimePast(); - // Decide whether to include witness transactions - // This is only needed in case the witness softfork activation is reverted - // (which would require a very deep reorganization). - // Note that the mempool would accept transactions with witness data before - // the deployment is active, but we would only ever mine blocks after activation - // unless there is a massive block reorganization with the witness softfork - // not activated. - // TODO: replace this with a call to main to assess validity of a mempool - // transaction (which in most cases can be a no-op). - fIncludeWitness = DeploymentActiveAfter(pindexPrev, chainparams.GetConsensus(), Consensus::DEPLOYMENT_SEGWIT); - int nPackagesSelected = 0; int nDescendantsUpdated = 0; addPackageTxs(nPackagesSelected, nDescendantsUpdated); @@ -215,17 +203,12 @@ bool BlockAssembler::TestPackage(uint64_t packageSize, int64_t packageSigOpsCost // Perform transaction-level checks before adding to block: // - transaction finality (locktime) -// - premature witness (in case segwit transactions are added to mempool before -// segwit activation) bool BlockAssembler::TestPackageTransactions(const CTxMemPool::setEntries& package) const { for (CTxMemPool::txiter it : package) { if (!IsFinalTx(it->GetTx(), nHeight, m_lock_time_cutoff)) { return false; } - if (!fIncludeWitness && it->GetTx().HasWitness()) { - return false; - } } return true; } diff --git a/src/node/miner.h b/src/node/miner.h index c96da874a7..5fd9abc280 100644 --- a/src/node/miner.h +++ b/src/node/miner.h @@ -45,7 +45,7 @@ struct CTxMemPoolModifiedEntry { nSigOpCostWithAncestors = entry->GetSigOpCostWithAncestors(); } - int64_t GetModifiedFee() const { return iter->GetModifiedFee(); } + CAmount GetModifiedFee() const { return iter->GetModifiedFee(); } uint64_t GetSizeWithAncestors() const { return nSizeWithAncestors; } CAmount GetModFeesWithAncestors() const { return nModFeesWithAncestors; } size_t GetTxSize() const { return iter->GetTxSize(); } @@ -132,7 +132,6 @@ private: std::unique_ptr<CBlockTemplate> pblocktemplate; // Configuration parameters for the block size - bool fIncludeWitness; unsigned int nBlockMaxWeight; CFeeRate blockMinFeeRate; diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index d6f80e1558..c6b884e40a 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -587,7 +587,7 @@ int GuiMain(int argc, char* argv[]) return EXIT_FAILURE; } #ifdef ENABLE_WALLET - // Parse URIs on command line -- this can affect Params() + // Parse URIs on command line PaymentServer::ipcParseCommandLine(argc, argv); #endif diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 7c22880dd1..85e3c23085 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -46,6 +46,7 @@ #include <QCursor> #include <QDateTime> #include <QDragEnterEvent> +#include <QKeySequence> #include <QListWidget> #include <QMenu> #include <QMenuBar> @@ -251,28 +252,28 @@ void BitcoinGUI::createActions() overviewAction->setStatusTip(tr("Show general overview of wallet")); overviewAction->setToolTip(overviewAction->statusTip()); overviewAction->setCheckable(true); - overviewAction->setShortcut(QKeySequence(Qt::ALT + Qt::Key_1)); + overviewAction->setShortcut(QKeySequence(QStringLiteral("Alt+1"))); tabGroup->addAction(overviewAction); sendCoinsAction = new QAction(platformStyle->SingleColorIcon(":/icons/send"), tr("&Send"), this); sendCoinsAction->setStatusTip(tr("Send coins to a Bitcoin address")); sendCoinsAction->setToolTip(sendCoinsAction->statusTip()); sendCoinsAction->setCheckable(true); - sendCoinsAction->setShortcut(QKeySequence(Qt::ALT + Qt::Key_2)); + sendCoinsAction->setShortcut(QKeySequence(QStringLiteral("Alt+2"))); tabGroup->addAction(sendCoinsAction); receiveCoinsAction = new QAction(platformStyle->SingleColorIcon(":/icons/receiving_addresses"), tr("&Receive"), this); receiveCoinsAction->setStatusTip(tr("Request payments (generates QR codes and bitcoin: URIs)")); receiveCoinsAction->setToolTip(receiveCoinsAction->statusTip()); receiveCoinsAction->setCheckable(true); - receiveCoinsAction->setShortcut(QKeySequence(Qt::ALT + Qt::Key_3)); + receiveCoinsAction->setShortcut(QKeySequence(QStringLiteral("Alt+3"))); tabGroup->addAction(receiveCoinsAction); historyAction = new QAction(platformStyle->SingleColorIcon(":/icons/history"), tr("&Transactions"), this); historyAction->setStatusTip(tr("Browse transaction history")); historyAction->setToolTip(historyAction->statusTip()); historyAction->setCheckable(true); - historyAction->setShortcut(QKeySequence(Qt::ALT + Qt::Key_4)); + historyAction->setShortcut(QKeySequence(QStringLiteral("Alt+4"))); tabGroup->addAction(historyAction); #ifdef ENABLE_WALLET @@ -290,7 +291,7 @@ void BitcoinGUI::createActions() quitAction = new QAction(tr("E&xit"), this); quitAction->setStatusTip(tr("Quit application")); - quitAction->setShortcut(QKeySequence(Qt::CTRL + Qt::Key_Q)); + quitAction->setShortcut(QKeySequence(tr("Ctrl+Q"))); quitAction->setMenuRole(QAction::QuitRole); aboutAction = new QAction(tr("&About %1").arg(PACKAGE_NAME), this); aboutAction->setStatusTip(tr("Show information about %1").arg(PACKAGE_NAME)); @@ -472,7 +473,7 @@ void BitcoinGUI::createMenuBar() QMenu* window_menu = appMenuBar->addMenu(tr("&Window")); QAction* minimize_action = window_menu->addAction(tr("&Minimize")); - minimize_action->setShortcut(QKeySequence(Qt::CTRL + Qt::Key_M)); + minimize_action->setShortcut(QKeySequence(tr("Ctrl+M"))); connect(minimize_action, &QAction::triggered, [] { QApplication::activeWindow()->showMinimized(); }); @@ -852,7 +853,7 @@ void BitcoinGUI::aboutClicked() if(!clientModel) return; - auto dlg = new HelpMessageDialog(this, /* about */ true); + auto dlg = new HelpMessageDialog(this, /*about=*/true); GUIUtil::ShowModalDialogAsynchronously(dlg); } diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index d7a2aaaf19..d3103492a4 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -16,10 +16,11 @@ #include <qt/platformstyle.h> #include <qt/walletmodel.h> -#include <wallet/coincontrol.h> #include <interfaces/node.h> #include <key_io.h> #include <policy/policy.h> +#include <wallet/coincontrol.h> +#include <wallet/coinselection.h> #include <wallet/wallet.h> #include <QApplication> @@ -32,7 +33,6 @@ #include <QTreeWidget> using wallet::CCoinControl; -using wallet::MIN_CHANGE; QList<CAmount> CoinControlDialog::payAmounts; bool CoinControlDialog::fSubtractFeeFromAmount = false; @@ -485,11 +485,10 @@ void CoinControlDialog::updateLabels(CCoinControl& m_coin_control, WalletModel * if (!CoinControlDialog::fSubtractFeeFromAmount) nChange -= nPayFee; - // Never create dust outputs; if we would, just add the dust to the fee. - if (nChange > 0 && nChange < MIN_CHANGE) - { + if (nChange > 0) { // Assumes a p2pkh script size CTxOut txout(nChange, CScript() << std::vector<unsigned char>(24, 0)); + // Never create dust outputs; if we would, just add the dust to the fee. if (IsDust(txout, model->node().getDustRelayFee())) { nPayFee += nChange; diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui index 2196801023..ead977296a 100644 --- a/src/qt/forms/debugwindow.ui +++ b/src/qt/forms/debugwindow.ui @@ -1594,10 +1594,10 @@ <item row="23" column="0"> <widget class="QLabel" name="peerAddrRelayEnabledLabel"> <property name="toolTip"> - <string extracomment="Tooltip text for the Address Relay field in the peer details area.">Whether we relay addresses to this peer.</string> + <string extracomment="Tooltip text for the Address Relay field in the peer details area, which displays whether we relay addresses to this peer (Yes/No).">Whether we relay addresses to this peer.</string> </property> <property name="text"> - <string>Address Relay</string> + <string extracomment="Text title for the Address Relay field in the peer details area, which displays whether we relay addresses to this peer (Yes/No).">Address Relay</string> </property> </widget> </item> @@ -1620,10 +1620,10 @@ <item row="24" column="0"> <widget class="QLabel" name="peerAddrProcessedLabel"> <property name="toolTip"> - <string extracomment="Tooltip text for the Addresses Processed field in the peer details area.">Total number of addresses processed, excluding those dropped due to rate-limiting.</string> + <string extracomment="Tooltip text for the Addresses Processed field in the peer details area, which displays the total number of addresses received from this peer that were processed (excludes addresses that were dropped due to rate-limiting).">The total number of addresses received from this peer that were processed (excludes addresses that were dropped due to rate-limiting).</string> </property> <property name="text"> - <string>Addresses Processed</string> + <string extracomment="Text title for the Addresses Processed field in the peer details area, which displays the total number of addresses received from this peer that were processed (excludes addresses that were dropped due to rate-limiting).">Addresses Processed</string> </property> </widget> </item> @@ -1646,10 +1646,10 @@ <item row="25" column="0"> <widget class="QLabel" name="peerAddrRateLimitedLabel"> <property name="toolTip"> - <string extracomment="Tooltip text for the Addresses Rate-Limited field in the peer details area.">Total number of addresses dropped due to rate-limiting.</string> + <string extracomment="Tooltip text for the Addresses Rate-Limited field in the peer details area, which displays the total number of addresses received from this peer that were dropped (not processed) due to rate-limiting.">The total number of addresses received from this peer that were dropped (not processed) due to rate-limiting.</string> </property> <property name="text"> - <string>Addresses Rate-Limited</string> + <string extracomment="Text title for the Addresses Rate-Limited field in the peer details area, which displays the total number of addresses received from this peer that were dropped (not processed) due to rate-limiting.">Addresses Rate-Limited</string> </property> </widget> </item> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 3108c93d7c..362601b512 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -47,6 +47,7 @@ #include <QGuiApplication> #include <QJsonObject> #include <QKeyEvent> +#include <QKeySequence> #include <QLatin1String> #include <QLineEdit> #include <QList> @@ -414,7 +415,7 @@ void bringToFront(QWidget* w) void handleCloseWindowShortcut(QWidget* w) { - QObject::connect(new QShortcut(QKeySequence(Qt::CTRL + Qt::Key_W), w), &QShortcut::activated, w, &QWidget::close); + QObject::connect(new QShortcut(QKeySequence(QObject::tr("Ctrl+W")), w), &QShortcut::activated, w, &QWidget::close); } void openDebugLogfile() diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index dcde7adec4..63b4055092 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -298,12 +298,12 @@ void Intro::setStatus(int status, const QString &message, quint64 bytesAvailable void Intro::UpdateFreeSpaceLabel() { - QString freeString = tr("%1 GB of space available").arg(m_bytes_available / GB_BYTES); + QString freeString = tr("%n GB of space available", "", m_bytes_available / GB_BYTES); if (m_bytes_available < m_required_space_gb * GB_BYTES) { - freeString += " " + tr("(of %1 GB needed)").arg(m_required_space_gb); + freeString += " " + tr("(of %n GB needed)", "", m_required_space_gb); ui->freeSpace->setStyleSheet("QLabel { color: #800000 }"); } else if (m_bytes_available / GB_BYTES - m_required_space_gb < 10) { - freeString += " " + tr("(%1 GB needed for full chain)").arg(m_required_space_gb); + freeString += " " + tr("(%n GB needed for full chain)", "", m_required_space_gb); ui->freeSpace->setStyleSheet("QLabel { color: #999900 }"); } else { ui->freeSpace->setStyleSheet(""); diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index 5d9ed5bf23..52bda59748 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -151,8 +151,28 @@ void OptionsModel::Init(bool resetSettings) if (!settings.contains("fListen")) settings.setValue("fListen", DEFAULT_LISTEN); - if (!gArgs.SoftSetBoolArg("-listen", settings.value("fListen").toBool())) + const bool listen{settings.value("fListen").toBool()}; + if (!gArgs.SoftSetBoolArg("-listen", listen)) { addOverriddenOption("-listen"); + } else if (!listen) { + // We successfully set -listen=0, thus mimic the logic from InitParameterInteraction(): + // "parameter interaction: -listen=0 -> setting -listenonion=0". + // + // Both -listen and -listenonion default to true. + // + // The call order is: + // + // InitParameterInteraction() + // would set -listenonion=0 if it sees -listen=0, but for bitcoin-qt with + // fListen=false -listen is 1 at this point + // + // OptionsModel::Init() + // (this method) can flip -listen from 1 to 0 if fListen=false + // + // AppInitParameterInteraction() + // raises an error if -listen=0 and -listenonion=1 + gArgs.SoftSetBoolArg("-listenonion", false); + } if (!settings.contains("server")) { settings.setValue("server", false); diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index cb09035b45..c82f0683fc 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -78,32 +78,11 @@ void PaymentServer::ipcParseCommandLine(int argc, char* argv[]) for (int i = 1; i < argc; i++) { QString arg(argv[i]); - if (arg.startsWith("-")) - continue; + if (arg.startsWith("-")) continue; - // If the bitcoin: URI contains a payment request, we are not able to detect the - // network as that would require fetching and parsing the payment request. - // That means clicking such an URI which contains a testnet payment request - // will start a mainnet instance and throw a "wrong network" error. if (arg.startsWith(BITCOIN_IPC_PREFIX, Qt::CaseInsensitive)) // bitcoin: URI { - if (savedPaymentRequests.contains(arg)) continue; savedPaymentRequests.insert(arg); - - SendCoinsRecipient r; - if (GUIUtil::parseBitcoinURI(arg, &r) && !r.address.isEmpty()) - { - auto tempChainParams = CreateChainParams(gArgs, CBaseChainParams::MAIN); - - if (IsValidDestinationString(r.address.toStdString(), *tempChainParams)) { - SelectParams(CBaseChainParams::MAIN); - } else { - tempChainParams = CreateChainParams(gArgs, CBaseChainParams::TESTNET); - if (IsValidDestinationString(r.address.toStdString(), *tempChainParams)) { - SelectParams(CBaseChainParams::TESTNET); - } - } - } } } } diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index cd3193a1d2..563bca76e5 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -80,7 +80,7 @@ QVariant PeerTableModel::data(const QModelIndex& index, int role) const //: An Outbound Connection to a Peer. tr("Outbound")); case ConnectionType: - return GUIUtil::ConnectionTypeToQString(rec->nodeStats.m_conn_type, /* prepend_direction */ false); + return GUIUtil::ConnectionTypeToQString(rec->nodeStats.m_conn_type, /*prepend_direction=*/false); case Network: return GUIUtil::NetworkToQString(rec->nodeStats.m_network); case Ping: diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index c5e5e69df6..dcc4f36aaa 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -40,6 +40,7 @@ #include <QDateTime> #include <QFont> #include <QKeyEvent> +#include <QKeySequence> #include <QLatin1String> #include <QLocale> #include <QMenu> @@ -847,7 +848,7 @@ void RPCConsole::setFontSize(int newSize) // clear console (reset icon sizes, default stylesheet) and re-add the content float oldPosFactor = 1.0 / ui->messagesWidget->verticalScrollBar()->maximum() * ui->messagesWidget->verticalScrollBar()->value(); - clear(/* keep_prompt */ true); + clear(/*keep_prompt=*/true); ui->messagesWidget->setHtml(str); ui->messagesWidget->verticalScrollBar()->setValue(oldPosFactor * ui->messagesWidget->verticalScrollBar()->maximum()); } @@ -1168,7 +1169,6 @@ void RPCConsole::updateDetailWidget() peerAddrDetails += "<br />" + tr("via %1").arg(QString::fromStdString(stats->nodeStats.addrLocal)); ui->peerHeading->setText(peerAddrDetails); ui->peerServices->setText(GUIUtil::formatServicesStr(stats->nodeStats.nServices)); - ui->peerRelayTxes->setText(stats->nodeStats.fRelayTxes ? ts.yes : ts.no); QString bip152_hb_settings; if (stats->nodeStats.m_bip152_highbandwidth_to) bip152_hb_settings = ts.to; if (stats->nodeStats.m_bip152_highbandwidth_from) bip152_hb_settings += (bip152_hb_settings.isEmpty() ? ts.from : QLatin1Char('/') + ts.from); @@ -1187,7 +1187,7 @@ void RPCConsole::updateDetailWidget() ui->timeoffset->setText(GUIUtil::formatTimeOffset(stats->nodeStats.nTimeOffset)); ui->peerVersion->setText(QString::number(stats->nodeStats.nVersion)); ui->peerSubversion->setText(QString::fromStdString(stats->nodeStats.cleanSubVer)); - ui->peerConnectionType->setText(GUIUtil::ConnectionTypeToQString(stats->nodeStats.m_conn_type, /* prepend_direction */ true)); + ui->peerConnectionType->setText(GUIUtil::ConnectionTypeToQString(stats->nodeStats.m_conn_type, /*prepend_direction=*/true)); ui->peerNetwork->setText(GUIUtil::NetworkToQString(stats->nodeStats.m_network)); if (stats->nodeStats.m_permissionFlags == NetPermissionFlags::None) { ui->peerPermissions->setText(ts.na); @@ -1220,6 +1220,7 @@ void RPCConsole::updateDetailWidget() ui->peerAddrRelayEnabled->setText(stats->nodeStateStats.m_addr_relay_enabled ? ts.yes : ts.no); ui->peerAddrProcessed->setText(QString::number(stats->nodeStateStats.m_addr_processed)); ui->peerAddrRateLimited->setText(QString::number(stats->nodeStateStats.m_addr_rate_limited)); + ui->peerRelayTxes->setText(stats->nodeStateStats.m_relay_txs ? ts.yes : ts.no); } ui->peersTabRightPanel->show(); @@ -1353,10 +1354,10 @@ QString RPCConsole::tabTitle(TabTypes tab_type) const QKeySequence RPCConsole::tabShortcut(TabTypes tab_type) const { switch (tab_type) { - case TabTypes::INFO: return QKeySequence(Qt::CTRL + Qt::Key_I); - case TabTypes::CONSOLE: return QKeySequence(Qt::CTRL + Qt::Key_T); - case TabTypes::GRAPH: return QKeySequence(Qt::CTRL + Qt::Key_N); - case TabTypes::PEERS: return QKeySequence(Qt::CTRL + Qt::Key_P); + case TabTypes::INFO: return QKeySequence(tr("Ctrl+I")); + case TabTypes::CONSOLE: return QKeySequence(tr("Ctrl+T")); + case TabTypes::GRAPH: return QKeySequence(tr("Ctrl+N")); + case TabTypes::PEERS: return QKeySequence(tr("Ctrl+P")); } // no default case, so the compiler can warn about missing cases assert(false); diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 579ef0c3fd..c924789796 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -401,6 +401,80 @@ bool SendCoinsDialog::PrepareSendText(QString& question_string, QString& informa return true; } +void SendCoinsDialog::presentPSBT(PartiallySignedTransaction& psbtx) +{ + // Serialize the PSBT + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << psbtx; + GUIUtil::setClipboard(EncodeBase64(ssTx.str()).c_str()); + QMessageBox msgBox; + msgBox.setText("Unsigned Transaction"); + msgBox.setInformativeText("The PSBT has been copied to the clipboard. You can also save it."); + msgBox.setStandardButtons(QMessageBox::Save | QMessageBox::Discard); + msgBox.setDefaultButton(QMessageBox::Discard); + switch (msgBox.exec()) { + case QMessageBox::Save: { + QString selectedFilter; + QString fileNameSuggestion = ""; + bool first = true; + for (const SendCoinsRecipient &rcp : m_current_transaction->getRecipients()) { + if (!first) { + fileNameSuggestion.append(" - "); + } + QString labelOrAddress = rcp.label.isEmpty() ? rcp.address : rcp.label; + QString amount = BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), rcp.amount); + fileNameSuggestion.append(labelOrAddress + "-" + amount); + first = false; + } + fileNameSuggestion.append(".psbt"); + QString filename = GUIUtil::getSaveFileName(this, + tr("Save Transaction Data"), fileNameSuggestion, + //: Expanded name of the binary PSBT file format. See: BIP 174. + tr("Partially Signed Transaction (Binary)") + QLatin1String(" (*.psbt)"), &selectedFilter); + if (filename.isEmpty()) { + return; + } + std::ofstream out{filename.toLocal8Bit().data(), std::ofstream::out | std::ofstream::binary}; + out << ssTx.str(); + out.close(); + Q_EMIT message(tr("PSBT saved"), "PSBT saved to disk", CClientUIInterface::MSG_INFORMATION); + break; + } + case QMessageBox::Discard: + break; + default: + assert(false); + } // msgBox.exec() +} + +bool SendCoinsDialog::signWithExternalSigner(PartiallySignedTransaction& psbtx, CMutableTransaction& mtx, bool& complete) { + TransactionError err; + try { + err = model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/true, /*bip32derivs=*/true, /*n_signed=*/nullptr, psbtx, complete); + } catch (const std::runtime_error& e) { + QMessageBox::critical(nullptr, tr("Sign failed"), e.what()); + return false; + } + if (err == TransactionError::EXTERNAL_SIGNER_NOT_FOUND) { + //: "External signer" means using devices such as hardware wallets. + QMessageBox::critical(nullptr, tr("External signer not found"), "External signer not found"); + return false; + } + if (err == TransactionError::EXTERNAL_SIGNER_FAILED) { + //: "External signer" means using devices such as hardware wallets. + QMessageBox::critical(nullptr, tr("External signer failure"), "External signer failure"); + return false; + } + if (err != TransactionError::OK) { + tfm::format(std::cerr, "Failed to sign PSBT"); + processSendCoinsReturn(WalletModel::TransactionCreationFailed); + return false; + } + // fillPSBT does not always properly finalize + complete = FinalizeAndExtractPSBT(psbtx, mtx); + return true; +} + void SendCoinsDialog::sendButtonClicked([[maybe_unused]] bool checked) { if(!model || !model->getOptionsModel()) @@ -411,7 +485,9 @@ void SendCoinsDialog::sendButtonClicked([[maybe_unused]] bool checked) assert(m_current_transaction); const QString confirmation = tr("Confirm send coins"); - auto confirmationDialog = new SendConfirmationDialog(confirmation, question_string, informative_text, detailed_text, SEND_CONFIRM_DELAY, !model->wallet().privateKeysDisabled(), model->getOptionsModel()->getEnablePSBTControls(), this); + const bool enable_send{!model->wallet().privateKeysDisabled() || model->wallet().hasExternalSigner()}; + const bool always_show_unsigned{model->getOptionsModel()->getEnablePSBTControls()}; + auto confirmationDialog = new SendConfirmationDialog(confirmation, question_string, informative_text, detailed_text, SEND_CONFIRM_DELAY, enable_send, always_show_unsigned, this); confirmationDialog->setAttribute(Qt::WA_DeleteOnClose); // TODO: Replace QDialog::exec() with safer QDialog::show(). const auto retval = static_cast<QMessageBox::StandardButton>(confirmationDialog->exec()); @@ -424,49 +500,50 @@ void SendCoinsDialog::sendButtonClicked([[maybe_unused]] bool checked) bool send_failure = false; if (retval == QMessageBox::Save) { + // "Create Unsigned" clicked CMutableTransaction mtx = CMutableTransaction{*(m_current_transaction->getWtx())}; PartiallySignedTransaction psbtx(mtx); bool complete = false; - // Always fill without signing first. This prevents an external signer - // from being called prematurely and is not expensive. - TransactionError err = model->wallet().fillPSBT(SIGHASH_ALL, false /* sign */, true /* bip32derivs */, nullptr, psbtx, complete); + // Fill without signing + TransactionError err = model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, /*n_signed=*/nullptr, psbtx, complete); assert(!complete); assert(err == TransactionError::OK); + + // Copy PSBT to clipboard and offer to save + presentPSBT(psbtx); + } else { + // "Send" clicked + assert(!model->wallet().privateKeysDisabled() || model->wallet().hasExternalSigner()); + bool broadcast = true; if (model->wallet().hasExternalSigner()) { - try { - err = model->wallet().fillPSBT(SIGHASH_ALL, true /* sign */, true /* bip32derivs */, nullptr, psbtx, complete); - } catch (const std::runtime_error& e) { - QMessageBox::critical(nullptr, tr("Sign failed"), e.what()); - send_failure = true; - return; - } - if (err == TransactionError::EXTERNAL_SIGNER_NOT_FOUND) { - //: "External signer" means using devices such as hardware wallets. - QMessageBox::critical(nullptr, tr("External signer not found"), "External signer not found"); - send_failure = true; - return; - } - if (err == TransactionError::EXTERNAL_SIGNER_FAILED) { - //: "External signer" means using devices such as hardware wallets. - QMessageBox::critical(nullptr, tr("External signer failure"), "External signer failure"); - send_failure = true; - return; - } - if (err != TransactionError::OK) { - tfm::format(std::cerr, "Failed to sign PSBT"); - processSendCoinsReturn(WalletModel::TransactionCreationFailed); - send_failure = true; - return; + CMutableTransaction mtx = CMutableTransaction{*(m_current_transaction->getWtx())}; + PartiallySignedTransaction psbtx(mtx); + bool complete = false; + // Always fill without signing first. This prevents an external signer + // from being called prematurely and is not expensive. + TransactionError err = model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, /*n_signed=*/nullptr, psbtx, complete); + assert(!complete); + assert(err == TransactionError::OK); + send_failure = !signWithExternalSigner(psbtx, mtx, complete); + // Don't broadcast when user rejects it on the device or there's a failure: + broadcast = complete && !send_failure; + if (!send_failure) { + // A transaction signed with an external signer is not always complete, + // e.g. in a multisig wallet. + if (complete) { + // Prepare transaction for broadcast transaction if complete + const CTransactionRef tx = MakeTransactionRef(mtx); + m_current_transaction->setWtx(tx); + } else { + presentPSBT(psbtx); + } } - // fillPSBT does not always properly finalize - complete = FinalizeAndExtractPSBT(psbtx, mtx); } - // Broadcast transaction if complete (even with an external signer this - // is not always the case, e.g. in a multisig wallet). - if (complete) { - const CTransactionRef tx = MakeTransactionRef(mtx); - m_current_transaction->setWtx(tx); + // Broadcast the transaction, unless an external signer was used and it + // failed, or more signatures are needed. + if (broadcast) { + // now send the prepared transaction WalletModel::SendCoinsReturn sendStatus = model->sendCoins(*m_current_transaction); // process sendStatus and on error generate message shown to user processSendCoinsReturn(sendStatus); @@ -476,64 +553,6 @@ void SendCoinsDialog::sendButtonClicked([[maybe_unused]] bool checked) } else { send_failure = true; } - return; - } - - // Copy PSBT to clipboard and offer to save - assert(!complete); - // Serialize the PSBT - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); - ssTx << psbtx; - GUIUtil::setClipboard(EncodeBase64(ssTx.str()).c_str()); - QMessageBox msgBox; - msgBox.setText("Unsigned Transaction"); - msgBox.setInformativeText("The PSBT has been copied to the clipboard. You can also save it."); - msgBox.setStandardButtons(QMessageBox::Save | QMessageBox::Discard); - msgBox.setDefaultButton(QMessageBox::Discard); - switch (msgBox.exec()) { - case QMessageBox::Save: { - QString selectedFilter; - QString fileNameSuggestion = ""; - bool first = true; - for (const SendCoinsRecipient &rcp : m_current_transaction->getRecipients()) { - if (!first) { - fileNameSuggestion.append(" - "); - } - QString labelOrAddress = rcp.label.isEmpty() ? rcp.address : rcp.label; - QString amount = BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), rcp.amount); - fileNameSuggestion.append(labelOrAddress + "-" + amount); - first = false; - } - fileNameSuggestion.append(".psbt"); - QString filename = GUIUtil::getSaveFileName(this, - tr("Save Transaction Data"), fileNameSuggestion, - //: Expanded name of the binary PSBT file format. See: BIP 174. - tr("Partially Signed Transaction (Binary)") + QLatin1String(" (*.psbt)"), &selectedFilter); - if (filename.isEmpty()) { - return; - } - std::ofstream out{filename.toLocal8Bit().data(), std::ofstream::out | std::ofstream::binary}; - out << ssTx.str(); - out.close(); - Q_EMIT message(tr("PSBT saved"), "PSBT saved to disk", CClientUIInterface::MSG_INFORMATION); - break; - } - case QMessageBox::Discard: - break; - default: - assert(false); - } // msgBox.exec() - } else { - assert(!model->wallet().privateKeysDisabled()); - // now send the prepared transaction - WalletModel::SendCoinsReturn sendStatus = model->sendCoins(*m_current_transaction); - // process sendStatus and on error generate message shown to user - processSendCoinsReturn(sendStatus); - - if (sendStatus.status == WalletModel::OK) { - Q_EMIT coinsSent(m_current_transaction->getWtx()->GetHash()); - } else { - send_failure = true; } } if (!send_failure) { diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index 4a16702756..400503d0c0 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -70,6 +70,8 @@ private: bool fFeeMinimized; const PlatformStyle *platformStyle; + // Copy PSBT to clipboard and offer to save it. + void presentPSBT(PartiallySignedTransaction& psbt); // Process WalletModel::SendCoinsReturn and generate a pair consisting // of a message and message flags for use in Q_EMIT message(). // Additional parameter msgArg can be used via .arg(msgArg). @@ -77,6 +79,15 @@ private: void minimizeFeeSection(bool fMinimize); // Format confirmation message bool PrepareSendText(QString& question_string, QString& informative_text, QString& detailed_text); + /* Sign PSBT using external signer. + * + * @param[in,out] psbtx the PSBT to sign + * @param[in,out] mtx needed to attempt to finalize + * @param[in,out] complete whether the PSBT is complete (a successfully signed multisig transaction may not be complete) + * + * @returns false if any failure occurred, which may include the user rejection of a transaction on the device. + */ + bool signWithExternalSigner(PartiallySignedTransaction& psbt, CMutableTransaction& mtx, bool& complete); void updateFeeMinimizedLabel(); void updateCoinControlState(); @@ -117,6 +128,8 @@ class SendConfirmationDialog : public QMessageBox public: SendConfirmationDialog(const QString& title, const QString& text, const QString& informative_text = "", const QString& detailed_text = "", int secDelay = SEND_CONFIRM_DELAY, bool enable_send = true, bool always_show_unsigned = true, QWidget* parent = nullptr); + /* Returns QMessageBox::Cancel, QMessageBox::Yes when "Send" is + clicked and QMessageBox::Save when "Create Unsigned" is clicked. */ int exec() override; private Q_SLOTS: diff --git a/src/qt/test/optiontests.cpp b/src/qt/test/optiontests.cpp index 51894e1915..4a943a6343 100644 --- a/src/qt/test/optiontests.cpp +++ b/src/qt/test/optiontests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <init.h> #include <qt/bitcoin.h> #include <qt/test/optiontests.h> #include <test/util/setup_common.h> @@ -29,3 +30,39 @@ void OptionTests::optionTests() }); gArgs.WriteSettingsFile(); } + +void OptionTests::parametersInteraction() +{ + // Test that the bug https://github.com/bitcoin-core/gui/issues/567 does not resurface. + // It was fixed via https://github.com/bitcoin-core/gui/pull/568. + // With fListen=false in ~/.config/Bitcoin/Bitcoin-Qt.conf and all else left as default, + // bitcoin-qt should set both -listen and -listenonion to false and start successfully. + gArgs.ClearPathCache(); + + gArgs.LockSettings([&](util::Settings& s) { + s.forced_settings.erase("listen"); + s.forced_settings.erase("listenonion"); + }); + QVERIFY(!gArgs.IsArgSet("-listen")); + QVERIFY(!gArgs.IsArgSet("-listenonion")); + + QSettings settings; + settings.setValue("fListen", false); + + OptionsModel{}; + + const bool expected{false}; + + QVERIFY(gArgs.IsArgSet("-listen")); + QCOMPARE(gArgs.GetBoolArg("-listen", !expected), expected); + + QVERIFY(gArgs.IsArgSet("-listenonion")); + QCOMPARE(gArgs.GetBoolArg("-listenonion", !expected), expected); + + QVERIFY(AppInitParameterInteraction(gArgs)); + + // cleanup + settings.remove("fListen"); + QVERIFY(!settings.contains("fListen")); + gArgs.ClearPathCache(); +} diff --git a/src/qt/test/optiontests.h b/src/qt/test/optiontests.h index 779d4cc209..39c1612c8f 100644 --- a/src/qt/test/optiontests.h +++ b/src/qt/test/optiontests.h @@ -17,6 +17,7 @@ public: private Q_SLOTS: void optionTests(); + void parametersInteraction(); private: interfaces::Node& m_node; diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 44b4fee2e7..6b0495f5a8 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -32,11 +32,11 @@ // Amount column is right-aligned it contains numbers static int column_alignments[] = { - Qt::AlignLeft|Qt::AlignVCenter, /* status */ - Qt::AlignLeft|Qt::AlignVCenter, /* watchonly */ - Qt::AlignLeft|Qt::AlignVCenter, /* date */ - Qt::AlignLeft|Qt::AlignVCenter, /* type */ - Qt::AlignLeft|Qt::AlignVCenter, /* address */ + Qt::AlignLeft|Qt::AlignVCenter, /*status=*/ + Qt::AlignLeft|Qt::AlignVCenter, /*watchonly=*/ + Qt::AlignLeft|Qt::AlignVCenter, /*date=*/ + Qt::AlignLeft|Qt::AlignVCenter, /*type=*/ + Qt::AlignLeft|Qt::AlignVCenter, /*address=*/ Qt::AlignRight|Qt::AlignVCenter /* amount */ }; diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 778ef04b77..47f3ba7e7f 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -550,7 +550,7 @@ void TransactionView::openThirdPartyTxUrl(QString url) QWidget *TransactionView::createDateRangeWidget() { dateRangeWidget = new QFrame(); - dateRangeWidget->setFrameStyle(QFrame::Panel | QFrame::Raised); + dateRangeWidget->setFrameStyle(static_cast<int>(QFrame::Panel) | static_cast<int>(QFrame::Raised)); dateRangeWidget->setContentsMargins(1,1,1,1); QHBoxLayout *layout = new QHBoxLayout(dateRangeWidget); layout->setContentsMargins(0,0,0,0); diff --git a/src/rest.cpp b/src/rest.cpp index 063872b47a..d59b6d1c13 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -15,6 +15,7 @@ #include <primitives/block.h> #include <primitives/transaction.h> #include <rpc/blockchain.h> +#include <rpc/mempool.h> #include <rpc/protocol.h> #include <rpc/server.h> #include <rpc/server_util.h> @@ -283,8 +284,8 @@ static bool rest_block(const std::any& context, return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); CBlock block; - CBlockIndex* pblockindex = nullptr; - CBlockIndex* tip = nullptr; + const CBlockIndex* pblockindex = nullptr; + const CBlockIndex* tip = nullptr; { ChainstateManager* maybe_chainman = GetChainman(context, req); if (!maybe_chainman) return false; @@ -669,7 +670,7 @@ static bool rest_tx(const std::any& context, HTTPRequest* req, const std::string case RetFormat::JSON: { UniValue objTx(UniValue::VOBJ); - TxToUniv(*tx, hashBlock, objTx); + TxToUniv(*tx, /*block_hash=*/hashBlock, /*entry=*/ objTx); std::string strJSON = objTx.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); @@ -854,7 +855,7 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: // include the script in a json output UniValue o(UniValue::VOBJ); - ScriptPubKeyToUniv(coin.out.scriptPubKey, o, true); + ScriptToUniv(coin.out.scriptPubKey, /*out=*/o, /*include_hex=*/true, /*include_address=*/true); utxo.pushKV("scriptPubKey", o); utxos.push_back(utxo); } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 86dfbbae35..cf72af1012 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -26,10 +26,6 @@ #include <node/coinstats.h> #include <node/context.h> #include <node/utxo_snapshot.h> -#include <policy/feerate.h> -#include <policy/fees.h> -#include <policy/policy.h> -#include <policy/rbf.h> #include <primitives/transaction.h> #include <rpc/server.h> #include <rpc/server_util.h> @@ -40,8 +36,8 @@ #include <txdb.h> #include <txmempool.h> #include <undo.h> +#include <univalue.h> #include <util/strencodings.h> -#include <util/string.h> #include <util/translation.h> #include <validation.h> #include <validationinterface.h> @@ -50,8 +46,6 @@ #include <stdint.h> -#include <univalue.h> - #include <condition_variable> #include <memory> #include <mutex> @@ -110,7 +104,8 @@ static int ComputeNextBlockAndDepth(const CBlockIndex* tip, const CBlockIndex* b return blockindex == tip ? 1 : -1; } -CBlockIndex* ParseHashOrHeight(const UniValue& param, ChainstateManager& chainman) { +static const CBlockIndex* ParseHashOrHeight(const UniValue& param, ChainstateManager& chainman) +{ LOCK(::cs_main); CChain& active_chain = chainman.ActiveChain(); @@ -127,7 +122,7 @@ CBlockIndex* ParseHashOrHeight(const UniValue& param, ChainstateManager& chainma return active_chain[height]; } else { const uint256 hash{ParseHashV(param, "hash_or_height")}; - CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(hash); + const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(hash); if (!pindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); @@ -192,7 +187,7 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIn // coinbase transaction (i.e. i == 0) doesn't have undo data const CTxUndo* txundo = (have_undo && i > 0) ? &blockUndo.vtxundo.at(i - 1) : nullptr; UniValue objTx(UniValue::VOBJ); - TxToUniv(*tx, uint256(), objTx, true, RPCSerializationFlags(), txundo, verbosity); + TxToUniv(*tx, /*block_hash=*/uint256(), /*entry=*/objTx, /*include_hex=*/true, RPCSerializationFlags(), txundo, verbosity); txs.push_back(objTx); } break; @@ -426,366 +421,6 @@ static RPCHelpMan getdifficulty() }; } -static std::vector<RPCResult> MempoolEntryDescription() { return { - RPCResult{RPCResult::Type::NUM, "vsize", "virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted."}, - RPCResult{RPCResult::Type::NUM, "weight", "transaction weight as defined in BIP 141."}, - RPCResult{RPCResult::Type::STR_AMOUNT, "fee", /*optional=*/true, - "transaction fee, denominated in " + CURRENCY_UNIT + " (DEPRECATED, returned only if config option -deprecatedrpc=fees is passed)"}, - RPCResult{RPCResult::Type::STR_AMOUNT, "modifiedfee", /*optional=*/true, - "transaction fee with fee deltas used for mining priority, denominated in " + CURRENCY_UNIT + - " (DEPRECATED, returned only if config option -deprecatedrpc=fees is passed)"}, - RPCResult{RPCResult::Type::NUM_TIME, "time", "local time transaction entered pool in seconds since 1 Jan 1970 GMT"}, - RPCResult{RPCResult::Type::NUM, "height", "block height when transaction entered pool"}, - RPCResult{RPCResult::Type::NUM, "descendantcount", "number of in-mempool descendant transactions (including this one)"}, - RPCResult{RPCResult::Type::NUM, "descendantsize", "virtual transaction size of in-mempool descendants (including this one)"}, - RPCResult{RPCResult::Type::STR_AMOUNT, "descendantfees", /*optional=*/true, - "transaction fees of in-mempool descendants (including this one) with fee deltas used for mining priority, denominated in " + - CURRENCY_ATOM + "s (DEPRECATED, returned only if config option -deprecatedrpc=fees is passed)"}, - RPCResult{RPCResult::Type::NUM, "ancestorcount", "number of in-mempool ancestor transactions (including this one)"}, - RPCResult{RPCResult::Type::NUM, "ancestorsize", "virtual transaction size of in-mempool ancestors (including this one)"}, - RPCResult{RPCResult::Type::STR_AMOUNT, "ancestorfees", /*optional=*/true, - "transaction fees of in-mempool ancestors (including this one) with fee deltas used for mining priority, denominated in " + - CURRENCY_ATOM + "s (DEPRECATED, returned only if config option -deprecatedrpc=fees is passed)"}, - RPCResult{RPCResult::Type::STR_HEX, "wtxid", "hash of serialized transaction, including witness data"}, - RPCResult{RPCResult::Type::OBJ, "fees", "", - { - RPCResult{RPCResult::Type::STR_AMOUNT, "base", "transaction fee, denominated in " + CURRENCY_UNIT}, - RPCResult{RPCResult::Type::STR_AMOUNT, "modified", "transaction fee with fee deltas used for mining priority, denominated in " + CURRENCY_UNIT}, - RPCResult{RPCResult::Type::STR_AMOUNT, "ancestor", "transaction fees of in-mempool ancestors (including this one) with fee deltas used for mining priority, denominated in " + CURRENCY_UNIT}, - RPCResult{RPCResult::Type::STR_AMOUNT, "descendant", "transaction fees of in-mempool descendants (including this one) with fee deltas used for mining priority, denominated in " + CURRENCY_UNIT}, - }}, - RPCResult{RPCResult::Type::ARR, "depends", "unconfirmed transactions used as inputs for this transaction", - {RPCResult{RPCResult::Type::STR_HEX, "transactionid", "parent transaction id"}}}, - RPCResult{RPCResult::Type::ARR, "spentby", "unconfirmed transactions spending outputs from this transaction", - {RPCResult{RPCResult::Type::STR_HEX, "transactionid", "child transaction id"}}}, - RPCResult{RPCResult::Type::BOOL, "bip125-replaceable", "Whether this transaction could be replaced due to BIP125 (replace-by-fee)"}, - RPCResult{RPCResult::Type::BOOL, "unbroadcast", "Whether this transaction is currently unbroadcast (initial broadcast not yet acknowledged by any peers)"}, -};} - -static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPoolEntry& e) EXCLUSIVE_LOCKS_REQUIRED(pool.cs) -{ - AssertLockHeld(pool.cs); - - info.pushKV("vsize", (int)e.GetTxSize()); - info.pushKV("weight", (int)e.GetTxWeight()); - // TODO: top-level fee fields are deprecated. deprecated_fee_fields_enabled blocks should be removed in v24 - const bool deprecated_fee_fields_enabled{IsDeprecatedRPCEnabled("fees")}; - if (deprecated_fee_fields_enabled) { - info.pushKV("fee", ValueFromAmount(e.GetFee())); - info.pushKV("modifiedfee", ValueFromAmount(e.GetModifiedFee())); - } - info.pushKV("time", count_seconds(e.GetTime())); - info.pushKV("height", (int)e.GetHeight()); - info.pushKV("descendantcount", e.GetCountWithDescendants()); - info.pushKV("descendantsize", e.GetSizeWithDescendants()); - if (deprecated_fee_fields_enabled) { - info.pushKV("descendantfees", e.GetModFeesWithDescendants()); - } - info.pushKV("ancestorcount", e.GetCountWithAncestors()); - info.pushKV("ancestorsize", e.GetSizeWithAncestors()); - if (deprecated_fee_fields_enabled) { - info.pushKV("ancestorfees", e.GetModFeesWithAncestors()); - } - info.pushKV("wtxid", pool.vTxHashes[e.vTxHashesIdx].first.ToString()); - - UniValue fees(UniValue::VOBJ); - fees.pushKV("base", ValueFromAmount(e.GetFee())); - fees.pushKV("modified", ValueFromAmount(e.GetModifiedFee())); - fees.pushKV("ancestor", ValueFromAmount(e.GetModFeesWithAncestors())); - fees.pushKV("descendant", ValueFromAmount(e.GetModFeesWithDescendants())); - info.pushKV("fees", fees); - - const CTransaction& tx = e.GetTx(); - std::set<std::string> setDepends; - for (const CTxIn& txin : tx.vin) - { - if (pool.exists(GenTxid::Txid(txin.prevout.hash))) - setDepends.insert(txin.prevout.hash.ToString()); - } - - UniValue depends(UniValue::VARR); - for (const std::string& dep : setDepends) - { - depends.push_back(dep); - } - - info.pushKV("depends", depends); - - UniValue spent(UniValue::VARR); - const CTxMemPool::txiter& it = pool.mapTx.find(tx.GetHash()); - const CTxMemPoolEntry::Children& children = it->GetMemPoolChildrenConst(); - for (const CTxMemPoolEntry& child : children) { - spent.push_back(child.GetTx().GetHash().ToString()); - } - - info.pushKV("spentby", spent); - - // Add opt-in RBF status - bool rbfStatus = false; - RBFTransactionState rbfState = IsRBFOptIn(tx, pool); - if (rbfState == RBFTransactionState::UNKNOWN) { - throw JSONRPCError(RPC_MISC_ERROR, "Transaction is not in mempool"); - } else if (rbfState == RBFTransactionState::REPLACEABLE_BIP125) { - rbfStatus = true; - } - - info.pushKV("bip125-replaceable", rbfStatus); - info.pushKV("unbroadcast", pool.IsUnbroadcastTx(tx.GetHash())); -} - -UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose, bool include_mempool_sequence) -{ - if (verbose) { - if (include_mempool_sequence) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Verbose results cannot contain mempool sequence values."); - } - LOCK(pool.cs); - UniValue o(UniValue::VOBJ); - for (const CTxMemPoolEntry& e : pool.mapTx) { - const uint256& hash = e.GetTx().GetHash(); - UniValue info(UniValue::VOBJ); - entryToJSON(pool, info, e); - // Mempool has unique entries so there is no advantage in using - // UniValue::pushKV, which checks if the key already exists in O(N). - // UniValue::__pushKV is used instead which currently is O(1). - o.__pushKV(hash.ToString(), info); - } - return o; - } else { - uint64_t mempool_sequence; - std::vector<uint256> vtxid; - { - LOCK(pool.cs); - pool.queryHashes(vtxid); - mempool_sequence = pool.GetSequence(); - } - UniValue a(UniValue::VARR); - for (const uint256& hash : vtxid) - a.push_back(hash.ToString()); - - if (!include_mempool_sequence) { - return a; - } else { - UniValue o(UniValue::VOBJ); - o.pushKV("txids", a); - o.pushKV("mempool_sequence", mempool_sequence); - return o; - } - } -} - -static RPCHelpMan getrawmempool() -{ - return RPCHelpMan{"getrawmempool", - "\nReturns all transaction ids in memory pool as a json array of string transaction ids.\n" - "\nHint: use getmempoolentry to fetch a specific transaction from the mempool.\n", - { - {"verbose", RPCArg::Type::BOOL, RPCArg::Default{false}, "True for a json object, false for array of transaction ids"}, - {"mempool_sequence", RPCArg::Type::BOOL, RPCArg::Default{false}, "If verbose=false, returns a json object with transaction list and mempool sequence number attached."}, - }, - { - RPCResult{"for verbose = false", - RPCResult::Type::ARR, "", "", - { - {RPCResult::Type::STR_HEX, "", "The transaction id"}, - }}, - RPCResult{"for verbose = true", - RPCResult::Type::OBJ_DYN, "", "", - { - {RPCResult::Type::OBJ, "transactionid", "", MempoolEntryDescription()}, - }}, - RPCResult{"for verbose = false and mempool_sequence = true", - RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::ARR, "txids", "", - { - {RPCResult::Type::STR_HEX, "", "The transaction id"}, - }}, - {RPCResult::Type::NUM, "mempool_sequence", "The mempool sequence value."}, - }}, - }, - RPCExamples{ - HelpExampleCli("getrawmempool", "true") - + HelpExampleRpc("getrawmempool", "true") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - bool fVerbose = false; - if (!request.params[0].isNull()) - fVerbose = request.params[0].get_bool(); - - bool include_mempool_sequence = false; - if (!request.params[1].isNull()) { - include_mempool_sequence = request.params[1].get_bool(); - } - - return MempoolToJSON(EnsureAnyMemPool(request.context), fVerbose, include_mempool_sequence); -}, - }; -} - -static RPCHelpMan getmempoolancestors() -{ - return RPCHelpMan{"getmempoolancestors", - "\nIf txid is in the mempool, returns all in-mempool ancestors.\n", - { - {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id (must be in mempool)"}, - {"verbose", RPCArg::Type::BOOL, RPCArg::Default{false}, "True for a json object, false for array of transaction ids"}, - }, - { - RPCResult{"for verbose = false", - RPCResult::Type::ARR, "", "", - {{RPCResult::Type::STR_HEX, "", "The transaction id of an in-mempool ancestor transaction"}}}, - RPCResult{"for verbose = true", - RPCResult::Type::OBJ_DYN, "", "", - { - {RPCResult::Type::OBJ, "transactionid", "", MempoolEntryDescription()}, - }}, - }, - RPCExamples{ - HelpExampleCli("getmempoolancestors", "\"mytxid\"") - + HelpExampleRpc("getmempoolancestors", "\"mytxid\"") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - bool fVerbose = false; - if (!request.params[1].isNull()) - fVerbose = request.params[1].get_bool(); - - uint256 hash = ParseHashV(request.params[0], "parameter 1"); - - const CTxMemPool& mempool = EnsureAnyMemPool(request.context); - LOCK(mempool.cs); - - CTxMemPool::txiter it = mempool.mapTx.find(hash); - if (it == mempool.mapTx.end()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); - } - - CTxMemPool::setEntries setAncestors; - uint64_t noLimit = std::numeric_limits<uint64_t>::max(); - std::string dummy; - mempool.CalculateMemPoolAncestors(*it, setAncestors, noLimit, noLimit, noLimit, noLimit, dummy, false); - - if (!fVerbose) { - UniValue o(UniValue::VARR); - for (CTxMemPool::txiter ancestorIt : setAncestors) { - o.push_back(ancestorIt->GetTx().GetHash().ToString()); - } - return o; - } else { - UniValue o(UniValue::VOBJ); - for (CTxMemPool::txiter ancestorIt : setAncestors) { - const CTxMemPoolEntry &e = *ancestorIt; - const uint256& _hash = e.GetTx().GetHash(); - UniValue info(UniValue::VOBJ); - entryToJSON(mempool, info, e); - o.pushKV(_hash.ToString(), info); - } - return o; - } -}, - }; -} - -static RPCHelpMan getmempooldescendants() -{ - return RPCHelpMan{"getmempooldescendants", - "\nIf txid is in the mempool, returns all in-mempool descendants.\n", - { - {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id (must be in mempool)"}, - {"verbose", RPCArg::Type::BOOL, RPCArg::Default{false}, "True for a json object, false for array of transaction ids"}, - }, - { - RPCResult{"for verbose = false", - RPCResult::Type::ARR, "", "", - {{RPCResult::Type::STR_HEX, "", "The transaction id of an in-mempool descendant transaction"}}}, - RPCResult{"for verbose = true", - RPCResult::Type::OBJ_DYN, "", "", - { - {RPCResult::Type::OBJ, "transactionid", "", MempoolEntryDescription()}, - }}, - }, - RPCExamples{ - HelpExampleCli("getmempooldescendants", "\"mytxid\"") - + HelpExampleRpc("getmempooldescendants", "\"mytxid\"") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - bool fVerbose = false; - if (!request.params[1].isNull()) - fVerbose = request.params[1].get_bool(); - - uint256 hash = ParseHashV(request.params[0], "parameter 1"); - - const CTxMemPool& mempool = EnsureAnyMemPool(request.context); - LOCK(mempool.cs); - - CTxMemPool::txiter it = mempool.mapTx.find(hash); - if (it == mempool.mapTx.end()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); - } - - CTxMemPool::setEntries setDescendants; - mempool.CalculateDescendants(it, setDescendants); - // CTxMemPool::CalculateDescendants will include the given tx - setDescendants.erase(it); - - if (!fVerbose) { - UniValue o(UniValue::VARR); - for (CTxMemPool::txiter descendantIt : setDescendants) { - o.push_back(descendantIt->GetTx().GetHash().ToString()); - } - - return o; - } else { - UniValue o(UniValue::VOBJ); - for (CTxMemPool::txiter descendantIt : setDescendants) { - const CTxMemPoolEntry &e = *descendantIt; - const uint256& _hash = e.GetTx().GetHash(); - UniValue info(UniValue::VOBJ); - entryToJSON(mempool, info, e); - o.pushKV(_hash.ToString(), info); - } - return o; - } -}, - }; -} - -static RPCHelpMan getmempoolentry() -{ - return RPCHelpMan{"getmempoolentry", - "\nReturns mempool data for given transaction\n", - { - {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id (must be in mempool)"}, - }, - RPCResult{ - RPCResult::Type::OBJ, "", "", MempoolEntryDescription()}, - RPCExamples{ - HelpExampleCli("getmempoolentry", "\"mytxid\"") - + HelpExampleRpc("getmempoolentry", "\"mytxid\"") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - uint256 hash = ParseHashV(request.params[0], "parameter 1"); - - const CTxMemPool& mempool = EnsureAnyMemPool(request.context); - LOCK(mempool.cs); - - CTxMemPool::txiter it = mempool.mapTx.find(hash); - if (it == mempool.mapTx.end()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); - } - - const CTxMemPoolEntry &e = *it; - UniValue info(UniValue::VOBJ); - entryToJSON(mempool, info, e); - return info; -}, - }; -} - static RPCHelpMan getblockfrompeer() { return RPCHelpMan{ @@ -854,7 +489,7 @@ static RPCHelpMan getblockhash() if (nHeight < 0 || nHeight > active_chain.Height()) throw JSONRPCError(RPC_INVALID_PARAMETER, "Block height out of range"); - CBlockIndex* pblockindex = active_chain[nHeight]; + const CBlockIndex* pblockindex = active_chain[nHeight]; return pblockindex->GetBlockHash().GetHex(); }, }; @@ -1036,7 +671,7 @@ static RPCHelpMan getblock() { {RPCResult::Type::STR, "asm", "The asm"}, {RPCResult::Type::STR, "hex", "The hex"}, - {RPCResult::Type::STR, "address", /* optional */ true, "The Bitcoin address (only if a well-defined address exists)"}, + {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, {RPCResult::Type::STR, "type", "The type (one of: " + GetAllOutputTypes() + ")"}, }}, }}, @@ -1132,7 +767,7 @@ static RPCHelpMan pruneblockchain() // too low to be a block time (corresponds to timestamp from Sep 2001). if (heightParam > 1000000000) { // Add a 2 hour buffer to include blocks which might have had old timestamps - CBlockIndex* pindex = active_chain.FindEarliestAtLeast(heightParam - TIMESTAMP_WINDOW, 0); + const CBlockIndex* pindex = active_chain.FindEarliestAtLeast(heightParam - TIMESTAMP_WINDOW, 0); if (!pindex) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Could not find block with at least the specified timestamp."); } @@ -1226,7 +861,7 @@ static RPCHelpMan gettxoutsetinfo() { UniValue ret(UniValue::VOBJ); - CBlockIndex* pindex{nullptr}; + const CBlockIndex* pindex{nullptr}; const CoinStatsHashType hash_type{request.params[0].isNull() ? CoinStatsHashType::HASH_SERIALIZED : ParseHashType(request.params[0].get_str())}; CCoinsStats stats{hash_type}; stats.index_requested = request.params[2].isNull() || request.params[2].get_bool(); @@ -1391,7 +1026,7 @@ static RPCHelpMan gettxout() } ret.pushKV("value", ValueFromAmount(coin.out.nValue)); UniValue o(UniValue::VOBJ); - ScriptPubKeyToUniv(coin.out.scriptPubKey, o, true); + ScriptToUniv(coin.out.scriptPubKey, /*out=*/o, /*include_hex=*/true, /*include_address=*/true); ret.pushKV("scriptPubKey", o); ret.pushKV("coinbase", (bool)coin.fCoinBase); @@ -1481,7 +1116,7 @@ static void SoftForkDescPushBack(const CBlockIndex* blockindex, UniValue& softfo // BIP9 status bip9.pushKV("status", get_state_name(current_state)); bip9.pushKV("since", g_versionbitscache.StateSinceHeight(blockindex->pprev, consensusParams, id)); - bip9.pushKV("status-next", get_state_name(next_state)); + bip9.pushKV("status_next", get_state_name(next_state)); // BIP9 signalling status, if applicable if (has_signal) { @@ -1527,38 +1162,38 @@ RPCHelpMan getblockchaininfo() { /* TODO: from v24, remove -deprecatedrpc=softforks */ return RPCHelpMan{"getblockchaininfo", - "Returns an object containing various state info regarding blockchain processing.\n", - {}, - RPCResult{ - RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR, "chain", "current network name (main, test, signet, regtest)"}, - {RPCResult::Type::NUM, "blocks", "the height of the most-work fully-validated chain. The genesis block has height 0"}, - {RPCResult::Type::NUM, "headers", "the current number of headers we have validated"}, - {RPCResult::Type::STR, "bestblockhash", "the hash of the currently best block"}, - {RPCResult::Type::NUM, "difficulty", "the current difficulty"}, - {RPCResult::Type::NUM_TIME, "time", "The block time expressed in " + UNIX_EPOCH_TIME}, - {RPCResult::Type::NUM_TIME, "mediantime", "The median block time expressed in " + UNIX_EPOCH_TIME}, - {RPCResult::Type::NUM, "verificationprogress", "estimate of verification progress [0..1]"}, - {RPCResult::Type::BOOL, "initialblockdownload", "(debug information) estimate of whether this node is in Initial Block Download mode"}, - {RPCResult::Type::STR_HEX, "chainwork", "total amount of work in active chain, in hexadecimal"}, - {RPCResult::Type::NUM, "size_on_disk", "the estimated size of the block and undo files on disk"}, - {RPCResult::Type::BOOL, "pruned", "if the blocks are subject to pruning"}, - {RPCResult::Type::NUM, "pruneheight", /*optional=*/true, "lowest-height complete block stored (only present if pruning is enabled)"}, - {RPCResult::Type::BOOL, "automatic_pruning", /*optional=*/true, "whether automatic pruning is enabled (only present if pruning is enabled)"}, - {RPCResult::Type::NUM, "prune_target_size", /*optional=*/true, "the target size used by pruning (only present if automatic pruning is enabled)"}, - {RPCResult::Type::OBJ_DYN, "softforks", "(DEPRECATED, returned only if config option -deprecatedrpc=softforks is passed) status of softforks", - { - {RPCResult::Type::OBJ, "xxxx", "name of the softfork", - RPCHelpForDeployment - }, - }}, - {RPCResult::Type::STR, "warnings", "any network and blockchain warnings"}, - }}, - RPCExamples{ - HelpExampleCli("getblockchaininfo", "") + "Returns an object containing various state info regarding blockchain processing.\n", + {}, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR, "chain", "current network name (main, test, signet, regtest)"}, + {RPCResult::Type::NUM, "blocks", "the height of the most-work fully-validated chain. The genesis block has height 0"}, + {RPCResult::Type::NUM, "headers", "the current number of headers we have validated"}, + {RPCResult::Type::STR, "bestblockhash", "the hash of the currently best block"}, + {RPCResult::Type::NUM, "difficulty", "the current difficulty"}, + {RPCResult::Type::NUM_TIME, "time", "The block time expressed in " + UNIX_EPOCH_TIME}, + {RPCResult::Type::NUM_TIME, "mediantime", "The median block time expressed in " + UNIX_EPOCH_TIME}, + {RPCResult::Type::NUM, "verificationprogress", "estimate of verification progress [0..1]"}, + {RPCResult::Type::BOOL, "initialblockdownload", "(debug information) estimate of whether this node is in Initial Block Download mode"}, + {RPCResult::Type::STR_HEX, "chainwork", "total amount of work in active chain, in hexadecimal"}, + {RPCResult::Type::NUM, "size_on_disk", "the estimated size of the block and undo files on disk"}, + {RPCResult::Type::BOOL, "pruned", "if the blocks are subject to pruning"}, + {RPCResult::Type::NUM, "pruneheight", /*optional=*/true, "lowest-height complete block stored (only present if pruning is enabled)"}, + {RPCResult::Type::BOOL, "automatic_pruning", /*optional=*/true, "whether automatic pruning is enabled (only present if pruning is enabled)"}, + {RPCResult::Type::NUM, "prune_target_size", /*optional=*/true, "the target size used by pruning (only present if automatic pruning is enabled)"}, + {RPCResult::Type::OBJ_DYN, "softforks", /*optional=*/true, "(DEPRECATED, returned only if config option -deprecatedrpc=softforks is passed) status of softforks", + { + {RPCResult::Type::OBJ, "xxxx", "name of the softfork", + RPCHelpForDeployment + }, + }}, + {RPCResult::Type::STR, "warnings", "any network and blockchain warnings"}, + }}, + RPCExamples{ + HelpExampleCli("getblockchaininfo", "") + HelpExampleRpc("getblockchaininfo", "") - }, + }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const ArgsManager& args{EnsureAnyArgsman(request.context)}; @@ -1623,7 +1258,7 @@ const std::vector<RPCResult> RPCHelpForDeployment{ {RPCResult::Type::NUM, "min_activation_height", "minimum height of blocks for which the rules may be enforced"}, {RPCResult::Type::STR, "status", "status of deployment at specified block (one of \"defined\", \"started\", \"locked_in\", \"active\", \"failed\")"}, {RPCResult::Type::NUM, "since", "height of the first block to which the status applies"}, - {RPCResult::Type::STR, "status-next", "status of deployment at the next block"}, + {RPCResult::Type::STR, "status_next", "status of deployment at the next block"}, {RPCResult::Type::OBJ, "statistics", /*optional=*/true, "numeric statistics about signalling for a softfork (only for \"started\" and \"locked_in\" status)", { {RPCResult::Type::NUM, "period", "the length in blocks of the signalling period"}, @@ -1632,7 +1267,7 @@ const std::vector<RPCResult> RPCHelpForDeployment{ {RPCResult::Type::NUM, "count", "the number of blocks with the version bit set in the current period"}, {RPCResult::Type::BOOL, "possible", /*optional=*/true, "returns false if there are not enough blocks left in this period to pass activation threshold (only for \"started\" status)"}, }}, - {RPCResult::Type::STR, "signalling", "indicates blocks that signalled with a # and blocks that did not with a -"}, + {RPCResult::Type::STR, "signalling", /*optional=*/true, "indicates blocks that signalled with a # and blocks that did not with a -"}, }}, }; @@ -1661,7 +1296,7 @@ static RPCHelpMan getdeploymentinfo() RPCResult::Type::OBJ, "", "", { {RPCResult::Type::STR, "hash", "requested block hash (or tip)"}, {RPCResult::Type::NUM, "height", "requested block height (or tip)"}, - {RPCResult::Type::OBJ, "deployments", "", { + {RPCResult::Type::OBJ_DYN, "deployments", "", { {RPCResult::Type::OBJ, "xxxx", "name of the deployment", RPCHelpForDeployment} }}, } @@ -1809,53 +1444,6 @@ static RPCHelpMan getchaintips() }; } -UniValue MempoolInfoToJSON(const CTxMemPool& pool) -{ - // Make sure this call is atomic in the pool. - LOCK(pool.cs); - UniValue ret(UniValue::VOBJ); - ret.pushKV("loaded", pool.IsLoaded()); - ret.pushKV("size", (int64_t)pool.size()); - ret.pushKV("bytes", (int64_t)pool.GetTotalTxSize()); - ret.pushKV("usage", (int64_t)pool.DynamicMemoryUsage()); - ret.pushKV("total_fee", ValueFromAmount(pool.GetTotalFee())); - size_t maxmempool = gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; - ret.pushKV("maxmempool", (int64_t) maxmempool); - ret.pushKV("mempoolminfee", ValueFromAmount(std::max(pool.GetMinFee(maxmempool), ::minRelayTxFee).GetFeePerK())); - ret.pushKV("minrelaytxfee", ValueFromAmount(::minRelayTxFee.GetFeePerK())); - ret.pushKV("unbroadcastcount", uint64_t{pool.GetUnbroadcastTxs().size()}); - return ret; -} - -static RPCHelpMan getmempoolinfo() -{ - return RPCHelpMan{"getmempoolinfo", - "\nReturns details on the active state of the TX memory pool.\n", - {}, - RPCResult{ - RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::BOOL, "loaded", "True if the mempool is fully loaded"}, - {RPCResult::Type::NUM, "size", "Current tx count"}, - {RPCResult::Type::NUM, "bytes", "Sum of all virtual transaction sizes as defined in BIP 141. Differs from actual serialized size because witness data is discounted"}, - {RPCResult::Type::NUM, "usage", "Total memory usage for the mempool"}, - {RPCResult::Type::STR_AMOUNT, "total_fee", "Total fees for the mempool in " + CURRENCY_UNIT + ", ignoring modified fees through prioritisetransaction"}, - {RPCResult::Type::NUM, "maxmempool", "Maximum memory usage for the mempool"}, - {RPCResult::Type::STR_AMOUNT, "mempoolminfee", "Minimum fee rate in " + CURRENCY_UNIT + "/kvB for tx to be accepted. Is the maximum of minrelaytxfee and minimum mempool fee"}, - {RPCResult::Type::STR_AMOUNT, "minrelaytxfee", "Current minimum relay fee for transactions"}, - {RPCResult::Type::NUM, "unbroadcastcount", "Current number of transactions that haven't passed initial broadcast yet"} - }}, - RPCExamples{ - HelpExampleCli("getmempoolinfo", "") - + HelpExampleRpc("getmempoolinfo", "") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - return MempoolInfoToJSON(EnsureAnyMemPool(request.context)); -}, - }; -} - static RPCHelpMan preciousblock() { return RPCHelpMan{"preciousblock", @@ -2177,7 +1765,7 @@ static RPCHelpMan getblockstats() { ChainstateManager& chainman = EnsureAnyChainman(request.context); LOCK(cs_main); - CBlockIndex* pindex{ParseHashOrHeight(request.params[0], chainman)}; + const CBlockIndex* pindex{ParseHashOrHeight(request.params[0], chainman)}; CHECK_NONFATAL(pindex != nullptr); std::set<std::string> stats; @@ -2352,41 +1940,6 @@ static RPCHelpMan getblockstats() }; } -static RPCHelpMan savemempool() -{ - return RPCHelpMan{"savemempool", - "\nDumps the mempool to disk. It will fail until the previous dump is fully loaded.\n", - {}, - RPCResult{ - RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR, "filename", "the directory and file where the mempool was saved"}, - }}, - RPCExamples{ - HelpExampleCli("savemempool", "") - + HelpExampleRpc("savemempool", "") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - const ArgsManager& args{EnsureAnyArgsman(request.context)}; - const CTxMemPool& mempool = EnsureAnyMemPool(request.context); - - if (!mempool.IsLoaded()) { - throw JSONRPCError(RPC_MISC_ERROR, "The mempool was not loaded yet"); - } - - if (!DumpMempool(mempool)) { - throw JSONRPCError(RPC_MISC_ERROR, "Unable to dump mempool to disk"); - } - - UniValue ret(UniValue::VOBJ); - ret.pushKV("filename", fs::path((args.GetDataDirNet() / "mempool.dat")).u8string()); - - return ret; -}, - }; -} - namespace { //! Search for a given set of pubkey scripts bool FindScriptPubKey(std::atomic<int>& scan_progress, const std::atomic<bool>& should_abort, int64_t& count, CCoinsViewCursor* cursor, const std::set<CScript>& needles, std::map<COutPoint, Coin>& out_results, std::function<void()>& interruption_point) @@ -2572,7 +2125,7 @@ static RPCHelpMan scantxoutset() g_should_abort_scan = false; int64_t count = 0; std::unique_ptr<CCoinsViewCursor> pcursor; - CBlockIndex* tip; + const CBlockIndex* tip; NodeContext& node = EnsureAnyNodeContext(request.context); { ChainstateManager& chainman = EnsureChainman(node); @@ -2760,7 +2313,7 @@ UniValue CreateUTXOSnapshot( { std::unique_ptr<CCoinsViewCursor> pcursor; CCoinsStats stats{CoinStatsHashType::HASH_SERIALIZED}; - CBlockIndex* tip; + const CBlockIndex* tip; { // We need to lock cs_main to ensure that the coinsdb isn't written to @@ -2825,6 +2378,7 @@ UniValue CreateUTXOSnapshot( return result; } + void RegisterBlockchainRPCCommands(CRPCTable &t) { // clang-format off @@ -2843,15 +2397,9 @@ static const CRPCCommand commands[] = { "blockchain", &getchaintips, }, { "blockchain", &getdifficulty, }, { "blockchain", &getdeploymentinfo, }, - { "blockchain", &getmempoolancestors, }, - { "blockchain", &getmempooldescendants, }, - { "blockchain", &getmempoolentry, }, - { "blockchain", &getmempoolinfo, }, - { "blockchain", &getrawmempool, }, { "blockchain", &gettxout, }, { "blockchain", &gettxoutsetinfo, }, { "blockchain", &pruneblockchain, }, - { "blockchain", &savemempool, }, { "blockchain", &verifychain, }, { "blockchain", &preciousblock, }, diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index 1f51d7c1ad..a8c6d171cc 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -20,7 +20,6 @@ extern RecursiveMutex cs_main; class CBlock; class CBlockIndex; class CChainState; -class CTxMemPool; class UniValue; namespace node { struct NodeContext; @@ -42,12 +41,6 @@ void RPCNotifyBlockChange(const CBlockIndex*); /** Block description to JSON */ UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, TxVerbosity verbosity) LOCKS_EXCLUDED(cs_main); -/** Mempool information to JSON */ -UniValue MempoolInfoToJSON(const CTxMemPool& pool); - -/** Mempool to JSON */ -UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose = false, bool include_mempool_sequence = false); - /** Block header to JSON */ UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex) LOCKS_EXCLUDED(cs_main); diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index c480a093a4..23e9d4074c 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -142,6 +142,10 @@ static const CRPCConvertParam vRPCConvertParams[] = { "send", 1, "conf_target" }, { "send", 3, "fee_rate"}, { "send", 4, "options" }, + { "sendall", 0, "recipients" }, + { "sendall", 1, "conf_target" }, + { "sendall", 3, "fee_rate"}, + { "sendall", 4, "options" }, { "importprivkey", 2, "rescan" }, { "importaddress", 2, "rescan" }, { "importaddress", 3, "p2sh" }, diff --git a/src/rpc/external_signer.cpp b/src/rpc/external_signer.cpp index 60ec15e904..82aa6f9516 100644 --- a/src/rpc/external_signer.cpp +++ b/src/rpc/external_signer.cpp @@ -22,7 +22,7 @@ static RPCHelpMan enumeratesigners() RPCResult{ RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::ARR, "signers", /* optional */ false, "", + {RPCResult::Type::ARR, "signers", /*optional=*/false, "", { {RPCResult::Type::OBJ, "", "", { diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp new file mode 100644 index 0000000000..1caf4ad96c --- /dev/null +++ b/src/rpc/mempool.cpp @@ -0,0 +1,689 @@ +// Copyright (c) 2010 Satoshi Nakamoto +// Copyright (c) 2009-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <rpc/blockchain.h> + +#include <core_io.h> +#include <fs.h> +#include <policy/rbf.h> +#include <primitives/transaction.h> +#include <rpc/server.h> +#include <rpc/server_util.h> +#include <rpc/util.h> +#include <txmempool.h> +#include <univalue.h> +#include <util/moneystr.h> +#include <validation.h> + +using node::DEFAULT_MAX_RAW_TX_FEE_RATE; +using node::NodeContext; + +static RPCHelpMan sendrawtransaction() +{ + return RPCHelpMan{"sendrawtransaction", + "\nSubmit a raw transaction (serialized, hex-encoded) to local node and network.\n" + "\nThe transaction will be sent unconditionally to all peers, so using sendrawtransaction\n" + "for manual rebroadcast may degrade privacy by leaking the transaction's origin, as\n" + "nodes will normally not rebroadcast non-wallet transactions already in their mempool.\n" + "\nA specific exception, RPC_TRANSACTION_ALREADY_IN_CHAIN, may throw if the transaction cannot be added to the mempool.\n" + "\nRelated RPCs: createrawtransaction, signrawtransactionwithkey\n", + { + {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex string of the raw transaction"}, + {"maxfeerate", RPCArg::Type::AMOUNT, RPCArg::Default{FormatMoney(DEFAULT_MAX_RAW_TX_FEE_RATE.GetFeePerK())}, + "Reject transactions whose fee rate is higher than the specified value, expressed in " + CURRENCY_UNIT + + "/kvB.\nSet to 0 to accept any fee rate.\n"}, + }, + RPCResult{ + RPCResult::Type::STR_HEX, "", "The transaction hash in hex" + }, + RPCExamples{ + "\nCreate a transaction\n" + + HelpExampleCli("createrawtransaction", "\"[{\\\"txid\\\" : \\\"mytxid\\\",\\\"vout\\\":0}]\" \"{\\\"myaddress\\\":0.01}\"") + + "Sign the transaction, and get back the hex\n" + + HelpExampleCli("signrawtransactionwithwallet", "\"myhex\"") + + "\nSend the transaction (signed hex)\n" + + HelpExampleCli("sendrawtransaction", "\"signedhex\"") + + "\nAs a JSON-RPC call\n" + + HelpExampleRpc("sendrawtransaction", "\"signedhex\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + RPCTypeCheck(request.params, { + UniValue::VSTR, + UniValueType(), // VNUM or VSTR, checked inside AmountFromValue() + }); + + CMutableTransaction mtx; + if (!DecodeHexTx(mtx, request.params[0].get_str())) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed. Make sure the tx has at least one input."); + } + CTransactionRef tx(MakeTransactionRef(std::move(mtx))); + + const CFeeRate max_raw_tx_fee_rate = request.params[1].isNull() ? + DEFAULT_MAX_RAW_TX_FEE_RATE : + CFeeRate(AmountFromValue(request.params[1])); + + int64_t virtual_size = GetVirtualTransactionSize(*tx); + CAmount max_raw_tx_fee = max_raw_tx_fee_rate.GetFee(virtual_size); + + std::string err_string; + AssertLockNotHeld(cs_main); + NodeContext& node = EnsureAnyNodeContext(request.context); + const TransactionError err = BroadcastTransaction(node, tx, err_string, max_raw_tx_fee, /*relay=*/true, /*wait_callback=*/true); + if (TransactionError::OK != err) { + throw JSONRPCTransactionError(err, err_string); + } + + return tx->GetHash().GetHex(); + }, + }; +} + +static RPCHelpMan testmempoolaccept() +{ + return RPCHelpMan{"testmempoolaccept", + "\nReturns result of mempool acceptance tests indicating if raw transaction(s) (serialized, hex-encoded) would be accepted by mempool.\n" + "\nIf multiple transactions are passed in, parents must come before children and package policies apply: the transactions cannot conflict with any mempool transactions or each other.\n" + "\nIf one transaction fails, other transactions may not be fully validated (the 'allowed' key will be blank).\n" + "\nThe maximum number of transactions allowed is " + ToString(MAX_PACKAGE_COUNT) + ".\n" + "\nThis checks if transactions violate the consensus or policy rules.\n" + "\nSee sendrawtransaction call.\n", + { + {"rawtxs", RPCArg::Type::ARR, RPCArg::Optional::NO, "An array of hex strings of raw transactions.", + { + {"rawtx", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, + }, + }, + {"maxfeerate", RPCArg::Type::AMOUNT, RPCArg::Default{FormatMoney(DEFAULT_MAX_RAW_TX_FEE_RATE.GetFeePerK())}, + "Reject transactions whose fee rate is higher than the specified value, expressed in " + CURRENCY_UNIT + "/kvB\n"}, + }, + RPCResult{ + RPCResult::Type::ARR, "", "The result of the mempool acceptance test for each raw transaction in the input array.\n" + "Returns results for each transaction in the same order they were passed in.\n" + "Transactions that cannot be fully validated due to failures in other transactions will not contain an 'allowed' result.\n", + { + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR_HEX, "txid", "The transaction hash in hex"}, + {RPCResult::Type::STR_HEX, "wtxid", "The transaction witness hash in hex"}, + {RPCResult::Type::STR, "package-error", /*optional=*/true, "Package validation error, if any (only possible if rawtxs had more than 1 transaction)."}, + {RPCResult::Type::BOOL, "allowed", /*optional=*/true, "Whether this tx would be accepted to the mempool and pass client-specified maxfeerate. " + "If not present, the tx was not fully validated due to a failure in another tx in the list."}, + {RPCResult::Type::NUM, "vsize", /*optional=*/true, "Virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted (only present when 'allowed' is true)"}, + {RPCResult::Type::OBJ, "fees", /*optional=*/true, "Transaction fees (only present if 'allowed' is true)", + { + {RPCResult::Type::STR_AMOUNT, "base", "transaction fee in " + CURRENCY_UNIT}, + }}, + {RPCResult::Type::STR, "reject-reason", /*optional=*/true, "Rejection string (only present when 'allowed' is false)"}, + }}, + } + }, + RPCExamples{ + "\nCreate a transaction\n" + + HelpExampleCli("createrawtransaction", "\"[{\\\"txid\\\" : \\\"mytxid\\\",\\\"vout\\\":0}]\" \"{\\\"myaddress\\\":0.01}\"") + + "Sign the transaction, and get back the hex\n" + + HelpExampleCli("signrawtransactionwithwallet", "\"myhex\"") + + "\nTest acceptance of the transaction (signed hex)\n" + + HelpExampleCli("testmempoolaccept", R"('["signedhex"]')") + + "\nAs a JSON-RPC call\n" + + HelpExampleRpc("testmempoolaccept", "[\"signedhex\"]") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + RPCTypeCheck(request.params, { + UniValue::VARR, + UniValueType(), // VNUM or VSTR, checked inside AmountFromValue() + }); + const UniValue raw_transactions = request.params[0].get_array(); + if (raw_transactions.size() < 1 || raw_transactions.size() > MAX_PACKAGE_COUNT) { + throw JSONRPCError(RPC_INVALID_PARAMETER, + "Array must contain between 1 and " + ToString(MAX_PACKAGE_COUNT) + " transactions."); + } + + const CFeeRate max_raw_tx_fee_rate = request.params[1].isNull() ? + DEFAULT_MAX_RAW_TX_FEE_RATE : + CFeeRate(AmountFromValue(request.params[1])); + + std::vector<CTransactionRef> txns; + txns.reserve(raw_transactions.size()); + for (const auto& rawtx : raw_transactions.getValues()) { + CMutableTransaction mtx; + if (!DecodeHexTx(mtx, rawtx.get_str())) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, + "TX decode failed: " + rawtx.get_str() + " Make sure the tx has at least one input."); + } + txns.emplace_back(MakeTransactionRef(std::move(mtx))); + } + + NodeContext& node = EnsureAnyNodeContext(request.context); + CTxMemPool& mempool = EnsureMemPool(node); + ChainstateManager& chainman = EnsureChainman(node); + CChainState& chainstate = chainman.ActiveChainstate(); + const PackageMempoolAcceptResult package_result = [&] { + LOCK(::cs_main); + if (txns.size() > 1) return ProcessNewPackage(chainstate, mempool, txns, /*test_accept=*/true); + return PackageMempoolAcceptResult(txns[0]->GetWitnessHash(), + chainman.ProcessTransaction(txns[0], /*test_accept=*/true)); + }(); + + UniValue rpc_result(UniValue::VARR); + // We will check transaction fees while we iterate through txns in order. If any transaction fee + // exceeds maxfeerate, we will leave the rest of the validation results blank, because it + // doesn't make sense to return a validation result for a transaction if its ancestor(s) would + // not be submitted. + bool exit_early{false}; + for (const auto& tx : txns) { + UniValue result_inner(UniValue::VOBJ); + result_inner.pushKV("txid", tx->GetHash().GetHex()); + result_inner.pushKV("wtxid", tx->GetWitnessHash().GetHex()); + if (package_result.m_state.GetResult() == PackageValidationResult::PCKG_POLICY) { + result_inner.pushKV("package-error", package_result.m_state.GetRejectReason()); + } + auto it = package_result.m_tx_results.find(tx->GetWitnessHash()); + if (exit_early || it == package_result.m_tx_results.end()) { + // Validation unfinished. Just return the txid and wtxid. + rpc_result.push_back(result_inner); + continue; + } + const auto& tx_result = it->second; + // Package testmempoolaccept doesn't allow transactions to already be in the mempool. + CHECK_NONFATAL(tx_result.m_result_type != MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + if (tx_result.m_result_type == MempoolAcceptResult::ResultType::VALID) { + const CAmount fee = tx_result.m_base_fees.value(); + // Check that fee does not exceed maximum fee + const int64_t virtual_size = tx_result.m_vsize.value(); + const CAmount max_raw_tx_fee = max_raw_tx_fee_rate.GetFee(virtual_size); + if (max_raw_tx_fee && fee > max_raw_tx_fee) { + result_inner.pushKV("allowed", false); + result_inner.pushKV("reject-reason", "max-fee-exceeded"); + exit_early = true; + } else { + // Only return the fee and vsize if the transaction would pass ATMP. + // These can be used to calculate the feerate. + result_inner.pushKV("allowed", true); + result_inner.pushKV("vsize", virtual_size); + UniValue fees(UniValue::VOBJ); + fees.pushKV("base", ValueFromAmount(fee)); + result_inner.pushKV("fees", fees); + } + } else { + result_inner.pushKV("allowed", false); + const TxValidationState state = tx_result.m_state; + if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { + result_inner.pushKV("reject-reason", "missing-inputs"); + } else { + result_inner.pushKV("reject-reason", state.GetRejectReason()); + } + } + rpc_result.push_back(result_inner); + } + return rpc_result; + }, + }; +} + +static std::vector<RPCResult> MempoolEntryDescription() +{ + return { + RPCResult{RPCResult::Type::NUM, "vsize", "virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted."}, + RPCResult{RPCResult::Type::NUM, "weight", "transaction weight as defined in BIP 141."}, + RPCResult{RPCResult::Type::STR_AMOUNT, "fee", /*optional=*/true, + "transaction fee, denominated in " + CURRENCY_UNIT + " (DEPRECATED, returned only if config option -deprecatedrpc=fees is passed)"}, + RPCResult{RPCResult::Type::STR_AMOUNT, "modifiedfee", /*optional=*/true, + "transaction fee with fee deltas used for mining priority, denominated in " + CURRENCY_UNIT + + " (DEPRECATED, returned only if config option -deprecatedrpc=fees is passed)"}, + RPCResult{RPCResult::Type::NUM_TIME, "time", "local time transaction entered pool in seconds since 1 Jan 1970 GMT"}, + RPCResult{RPCResult::Type::NUM, "height", "block height when transaction entered pool"}, + RPCResult{RPCResult::Type::NUM, "descendantcount", "number of in-mempool descendant transactions (including this one)"}, + RPCResult{RPCResult::Type::NUM, "descendantsize", "virtual transaction size of in-mempool descendants (including this one)"}, + RPCResult{RPCResult::Type::STR_AMOUNT, "descendantfees", /*optional=*/true, + "transaction fees of in-mempool descendants (including this one) with fee deltas used for mining priority, denominated in " + + CURRENCY_ATOM + "s (DEPRECATED, returned only if config option -deprecatedrpc=fees is passed)"}, + RPCResult{RPCResult::Type::NUM, "ancestorcount", "number of in-mempool ancestor transactions (including this one)"}, + RPCResult{RPCResult::Type::NUM, "ancestorsize", "virtual transaction size of in-mempool ancestors (including this one)"}, + RPCResult{RPCResult::Type::STR_AMOUNT, "ancestorfees", /*optional=*/true, + "transaction fees of in-mempool ancestors (including this one) with fee deltas used for mining priority, denominated in " + + CURRENCY_ATOM + "s (DEPRECATED, returned only if config option -deprecatedrpc=fees is passed)"}, + RPCResult{RPCResult::Type::STR_HEX, "wtxid", "hash of serialized transaction, including witness data"}, + RPCResult{RPCResult::Type::OBJ, "fees", "", + { + RPCResult{RPCResult::Type::STR_AMOUNT, "base", "transaction fee, denominated in " + CURRENCY_UNIT}, + RPCResult{RPCResult::Type::STR_AMOUNT, "modified", "transaction fee with fee deltas used for mining priority, denominated in " + CURRENCY_UNIT}, + RPCResult{RPCResult::Type::STR_AMOUNT, "ancestor", "transaction fees of in-mempool ancestors (including this one) with fee deltas used for mining priority, denominated in " + CURRENCY_UNIT}, + RPCResult{RPCResult::Type::STR_AMOUNT, "descendant", "transaction fees of in-mempool descendants (including this one) with fee deltas used for mining priority, denominated in " + CURRENCY_UNIT}, + }}, + RPCResult{RPCResult::Type::ARR, "depends", "unconfirmed transactions used as inputs for this transaction", + {RPCResult{RPCResult::Type::STR_HEX, "transactionid", "parent transaction id"}}}, + RPCResult{RPCResult::Type::ARR, "spentby", "unconfirmed transactions spending outputs from this transaction", + {RPCResult{RPCResult::Type::STR_HEX, "transactionid", "child transaction id"}}}, + RPCResult{RPCResult::Type::BOOL, "bip125-replaceable", "Whether this transaction could be replaced due to BIP125 (replace-by-fee)"}, + RPCResult{RPCResult::Type::BOOL, "unbroadcast", "Whether this transaction is currently unbroadcast (initial broadcast not yet acknowledged by any peers)"}, + }; +} + +static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPoolEntry& e) EXCLUSIVE_LOCKS_REQUIRED(pool.cs) +{ + AssertLockHeld(pool.cs); + + info.pushKV("vsize", (int)e.GetTxSize()); + info.pushKV("weight", (int)e.GetTxWeight()); + // TODO: top-level fee fields are deprecated. deprecated_fee_fields_enabled blocks should be removed in v24 + const bool deprecated_fee_fields_enabled{IsDeprecatedRPCEnabled("fees")}; + if (deprecated_fee_fields_enabled) { + info.pushKV("fee", ValueFromAmount(e.GetFee())); + info.pushKV("modifiedfee", ValueFromAmount(e.GetModifiedFee())); + } + info.pushKV("time", count_seconds(e.GetTime())); + info.pushKV("height", (int)e.GetHeight()); + info.pushKV("descendantcount", e.GetCountWithDescendants()); + info.pushKV("descendantsize", e.GetSizeWithDescendants()); + if (deprecated_fee_fields_enabled) { + info.pushKV("descendantfees", e.GetModFeesWithDescendants()); + } + info.pushKV("ancestorcount", e.GetCountWithAncestors()); + info.pushKV("ancestorsize", e.GetSizeWithAncestors()); + if (deprecated_fee_fields_enabled) { + info.pushKV("ancestorfees", e.GetModFeesWithAncestors()); + } + info.pushKV("wtxid", pool.vTxHashes[e.vTxHashesIdx].first.ToString()); + + UniValue fees(UniValue::VOBJ); + fees.pushKV("base", ValueFromAmount(e.GetFee())); + fees.pushKV("modified", ValueFromAmount(e.GetModifiedFee())); + fees.pushKV("ancestor", ValueFromAmount(e.GetModFeesWithAncestors())); + fees.pushKV("descendant", ValueFromAmount(e.GetModFeesWithDescendants())); + info.pushKV("fees", fees); + + const CTransaction& tx = e.GetTx(); + std::set<std::string> setDepends; + for (const CTxIn& txin : tx.vin) + { + if (pool.exists(GenTxid::Txid(txin.prevout.hash))) + setDepends.insert(txin.prevout.hash.ToString()); + } + + UniValue depends(UniValue::VARR); + for (const std::string& dep : setDepends) + { + depends.push_back(dep); + } + + info.pushKV("depends", depends); + + UniValue spent(UniValue::VARR); + const CTxMemPool::txiter& it = pool.mapTx.find(tx.GetHash()); + const CTxMemPoolEntry::Children& children = it->GetMemPoolChildrenConst(); + for (const CTxMemPoolEntry& child : children) { + spent.push_back(child.GetTx().GetHash().ToString()); + } + + info.pushKV("spentby", spent); + + // Add opt-in RBF status + bool rbfStatus = false; + RBFTransactionState rbfState = IsRBFOptIn(tx, pool); + if (rbfState == RBFTransactionState::UNKNOWN) { + throw JSONRPCError(RPC_MISC_ERROR, "Transaction is not in mempool"); + } else if (rbfState == RBFTransactionState::REPLACEABLE_BIP125) { + rbfStatus = true; + } + + info.pushKV("bip125-replaceable", rbfStatus); + info.pushKV("unbroadcast", pool.IsUnbroadcastTx(tx.GetHash())); +} + +UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose, bool include_mempool_sequence) +{ + if (verbose) { + if (include_mempool_sequence) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Verbose results cannot contain mempool sequence values."); + } + LOCK(pool.cs); + UniValue o(UniValue::VOBJ); + for (const CTxMemPoolEntry& e : pool.mapTx) { + const uint256& hash = e.GetTx().GetHash(); + UniValue info(UniValue::VOBJ); + entryToJSON(pool, info, e); + // Mempool has unique entries so there is no advantage in using + // UniValue::pushKV, which checks if the key already exists in O(N). + // UniValue::__pushKV is used instead which currently is O(1). + o.__pushKV(hash.ToString(), info); + } + return o; + } else { + uint64_t mempool_sequence; + std::vector<uint256> vtxid; + { + LOCK(pool.cs); + pool.queryHashes(vtxid); + mempool_sequence = pool.GetSequence(); + } + UniValue a(UniValue::VARR); + for (const uint256& hash : vtxid) + a.push_back(hash.ToString()); + + if (!include_mempool_sequence) { + return a; + } else { + UniValue o(UniValue::VOBJ); + o.pushKV("txids", a); + o.pushKV("mempool_sequence", mempool_sequence); + return o; + } + } +} + +static RPCHelpMan getrawmempool() +{ + return RPCHelpMan{"getrawmempool", + "\nReturns all transaction ids in memory pool as a json array of string transaction ids.\n" + "\nHint: use getmempoolentry to fetch a specific transaction from the mempool.\n", + { + {"verbose", RPCArg::Type::BOOL, RPCArg::Default{false}, "True for a json object, false for array of transaction ids"}, + {"mempool_sequence", RPCArg::Type::BOOL, RPCArg::Default{false}, "If verbose=false, returns a json object with transaction list and mempool sequence number attached."}, + }, + { + RPCResult{"for verbose = false", + RPCResult::Type::ARR, "", "", + { + {RPCResult::Type::STR_HEX, "", "The transaction id"}, + }}, + RPCResult{"for verbose = true", + RPCResult::Type::OBJ_DYN, "", "", + { + {RPCResult::Type::OBJ, "transactionid", "", MempoolEntryDescription()}, + }}, + RPCResult{"for verbose = false and mempool_sequence = true", + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::ARR, "txids", "", + { + {RPCResult::Type::STR_HEX, "", "The transaction id"}, + }}, + {RPCResult::Type::NUM, "mempool_sequence", "The mempool sequence value."}, + }}, + }, + RPCExamples{ + HelpExampleCli("getrawmempool", "true") + + HelpExampleRpc("getrawmempool", "true") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + bool fVerbose = false; + if (!request.params[0].isNull()) + fVerbose = request.params[0].get_bool(); + + bool include_mempool_sequence = false; + if (!request.params[1].isNull()) { + include_mempool_sequence = request.params[1].get_bool(); + } + + return MempoolToJSON(EnsureAnyMemPool(request.context), fVerbose, include_mempool_sequence); +}, + }; +} + +static RPCHelpMan getmempoolancestors() +{ + return RPCHelpMan{"getmempoolancestors", + "\nIf txid is in the mempool, returns all in-mempool ancestors.\n", + { + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id (must be in mempool)"}, + {"verbose", RPCArg::Type::BOOL, RPCArg::Default{false}, "True for a json object, false for array of transaction ids"}, + }, + { + RPCResult{"for verbose = false", + RPCResult::Type::ARR, "", "", + {{RPCResult::Type::STR_HEX, "", "The transaction id of an in-mempool ancestor transaction"}}}, + RPCResult{"for verbose = true", + RPCResult::Type::OBJ_DYN, "", "", + { + {RPCResult::Type::OBJ, "transactionid", "", MempoolEntryDescription()}, + }}, + }, + RPCExamples{ + HelpExampleCli("getmempoolancestors", "\"mytxid\"") + + HelpExampleRpc("getmempoolancestors", "\"mytxid\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + bool fVerbose = false; + if (!request.params[1].isNull()) + fVerbose = request.params[1].get_bool(); + + uint256 hash = ParseHashV(request.params[0], "parameter 1"); + + const CTxMemPool& mempool = EnsureAnyMemPool(request.context); + LOCK(mempool.cs); + + CTxMemPool::txiter it = mempool.mapTx.find(hash); + if (it == mempool.mapTx.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); + } + + CTxMemPool::setEntries setAncestors; + uint64_t noLimit = std::numeric_limits<uint64_t>::max(); + std::string dummy; + mempool.CalculateMemPoolAncestors(*it, setAncestors, noLimit, noLimit, noLimit, noLimit, dummy, false); + + if (!fVerbose) { + UniValue o(UniValue::VARR); + for (CTxMemPool::txiter ancestorIt : setAncestors) { + o.push_back(ancestorIt->GetTx().GetHash().ToString()); + } + return o; + } else { + UniValue o(UniValue::VOBJ); + for (CTxMemPool::txiter ancestorIt : setAncestors) { + const CTxMemPoolEntry &e = *ancestorIt; + const uint256& _hash = e.GetTx().GetHash(); + UniValue info(UniValue::VOBJ); + entryToJSON(mempool, info, e); + o.pushKV(_hash.ToString(), info); + } + return o; + } +}, + }; +} + +static RPCHelpMan getmempooldescendants() +{ + return RPCHelpMan{"getmempooldescendants", + "\nIf txid is in the mempool, returns all in-mempool descendants.\n", + { + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id (must be in mempool)"}, + {"verbose", RPCArg::Type::BOOL, RPCArg::Default{false}, "True for a json object, false for array of transaction ids"}, + }, + { + RPCResult{"for verbose = false", + RPCResult::Type::ARR, "", "", + {{RPCResult::Type::STR_HEX, "", "The transaction id of an in-mempool descendant transaction"}}}, + RPCResult{"for verbose = true", + RPCResult::Type::OBJ_DYN, "", "", + { + {RPCResult::Type::OBJ, "transactionid", "", MempoolEntryDescription()}, + }}, + }, + RPCExamples{ + HelpExampleCli("getmempooldescendants", "\"mytxid\"") + + HelpExampleRpc("getmempooldescendants", "\"mytxid\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + bool fVerbose = false; + if (!request.params[1].isNull()) + fVerbose = request.params[1].get_bool(); + + uint256 hash = ParseHashV(request.params[0], "parameter 1"); + + const CTxMemPool& mempool = EnsureAnyMemPool(request.context); + LOCK(mempool.cs); + + CTxMemPool::txiter it = mempool.mapTx.find(hash); + if (it == mempool.mapTx.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); + } + + CTxMemPool::setEntries setDescendants; + mempool.CalculateDescendants(it, setDescendants); + // CTxMemPool::CalculateDescendants will include the given tx + setDescendants.erase(it); + + if (!fVerbose) { + UniValue o(UniValue::VARR); + for (CTxMemPool::txiter descendantIt : setDescendants) { + o.push_back(descendantIt->GetTx().GetHash().ToString()); + } + + return o; + } else { + UniValue o(UniValue::VOBJ); + for (CTxMemPool::txiter descendantIt : setDescendants) { + const CTxMemPoolEntry &e = *descendantIt; + const uint256& _hash = e.GetTx().GetHash(); + UniValue info(UniValue::VOBJ); + entryToJSON(mempool, info, e); + o.pushKV(_hash.ToString(), info); + } + return o; + } +}, + }; +} + +static RPCHelpMan getmempoolentry() +{ + return RPCHelpMan{"getmempoolentry", + "\nReturns mempool data for given transaction\n", + { + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id (must be in mempool)"}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", MempoolEntryDescription()}, + RPCExamples{ + HelpExampleCli("getmempoolentry", "\"mytxid\"") + + HelpExampleRpc("getmempoolentry", "\"mytxid\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + uint256 hash = ParseHashV(request.params[0], "parameter 1"); + + const CTxMemPool& mempool = EnsureAnyMemPool(request.context); + LOCK(mempool.cs); + + CTxMemPool::txiter it = mempool.mapTx.find(hash); + if (it == mempool.mapTx.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); + } + + const CTxMemPoolEntry &e = *it; + UniValue info(UniValue::VOBJ); + entryToJSON(mempool, info, e); + return info; +}, + }; +} + +UniValue MempoolInfoToJSON(const CTxMemPool& pool) +{ + // Make sure this call is atomic in the pool. + LOCK(pool.cs); + UniValue ret(UniValue::VOBJ); + ret.pushKV("loaded", pool.IsLoaded()); + ret.pushKV("size", (int64_t)pool.size()); + ret.pushKV("bytes", (int64_t)pool.GetTotalTxSize()); + ret.pushKV("usage", (int64_t)pool.DynamicMemoryUsage()); + ret.pushKV("total_fee", ValueFromAmount(pool.GetTotalFee())); + int64_t maxmempool{gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000}; + ret.pushKV("maxmempool", maxmempool); + ret.pushKV("mempoolminfee", ValueFromAmount(std::max(pool.GetMinFee(maxmempool), ::minRelayTxFee).GetFeePerK())); + ret.pushKV("minrelaytxfee", ValueFromAmount(::minRelayTxFee.GetFeePerK())); + ret.pushKV("unbroadcastcount", uint64_t{pool.GetUnbroadcastTxs().size()}); + return ret; +} + +static RPCHelpMan getmempoolinfo() +{ + return RPCHelpMan{"getmempoolinfo", + "\nReturns details on the active state of the TX memory pool.\n", + {}, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::BOOL, "loaded", "True if the mempool is fully loaded"}, + {RPCResult::Type::NUM, "size", "Current tx count"}, + {RPCResult::Type::NUM, "bytes", "Sum of all virtual transaction sizes as defined in BIP 141. Differs from actual serialized size because witness data is discounted"}, + {RPCResult::Type::NUM, "usage", "Total memory usage for the mempool"}, + {RPCResult::Type::STR_AMOUNT, "total_fee", "Total fees for the mempool in " + CURRENCY_UNIT + ", ignoring modified fees through prioritisetransaction"}, + {RPCResult::Type::NUM, "maxmempool", "Maximum memory usage for the mempool"}, + {RPCResult::Type::STR_AMOUNT, "mempoolminfee", "Minimum fee rate in " + CURRENCY_UNIT + "/kvB for tx to be accepted. Is the maximum of minrelaytxfee and minimum mempool fee"}, + {RPCResult::Type::STR_AMOUNT, "minrelaytxfee", "Current minimum relay fee for transactions"}, + {RPCResult::Type::NUM, "unbroadcastcount", "Current number of transactions that haven't passed initial broadcast yet"} + }}, + RPCExamples{ + HelpExampleCli("getmempoolinfo", "") + + HelpExampleRpc("getmempoolinfo", "") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + return MempoolInfoToJSON(EnsureAnyMemPool(request.context)); +}, + }; +} + +static RPCHelpMan savemempool() +{ + return RPCHelpMan{"savemempool", + "\nDumps the mempool to disk. It will fail until the previous dump is fully loaded.\n", + {}, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR, "filename", "the directory and file where the mempool was saved"}, + }}, + RPCExamples{ + HelpExampleCli("savemempool", "") + + HelpExampleRpc("savemempool", "") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + const ArgsManager& args{EnsureAnyArgsman(request.context)}; + const CTxMemPool& mempool = EnsureAnyMemPool(request.context); + + if (!mempool.IsLoaded()) { + throw JSONRPCError(RPC_MISC_ERROR, "The mempool was not loaded yet"); + } + + if (!DumpMempool(mempool)) { + throw JSONRPCError(RPC_MISC_ERROR, "Unable to dump mempool to disk"); + } + + UniValue ret(UniValue::VOBJ); + ret.pushKV("filename", fs::path((args.GetDataDirNet() / "mempool.dat")).u8string()); + + return ret; +}, + }; +} + +void RegisterMempoolRPCCommands(CRPCTable& t) +{ + static const CRPCCommand commands[]{ + // category actor (function) + // -------- ---------------- + {"rawtransactions", &sendrawtransaction}, + {"rawtransactions", &testmempoolaccept}, + {"blockchain", &getmempoolancestors}, + {"blockchain", &getmempooldescendants}, + {"blockchain", &getmempoolentry}, + {"blockchain", &getmempoolinfo}, + {"blockchain", &getrawmempool}, + {"blockchain", &savemempool}, + }; + for (const auto& c : commands) { + t.appendCommand(c.name, &c); + } +} diff --git a/src/rpc/mempool.h b/src/rpc/mempool.h new file mode 100644 index 0000000000..229d7d52dd --- /dev/null +++ b/src/rpc/mempool.h @@ -0,0 +1,17 @@ +// Copyright (c) 2017-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_RPC_MEMPOOL_H +#define BITCOIN_RPC_MEMPOOL_H + +class CTxMemPool; +class UniValue; + +/** Mempool information to JSON */ +UniValue MempoolInfoToJSON(const CTxMemPool& pool); + +/** Mempool to JSON */ +UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose = false, bool include_mempool_sequence = false); + +#endif // BITCOIN_RPC_MEMPOOL_H diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 8d7b48d697..89f096981f 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -116,7 +116,7 @@ static RPCHelpMan createmultisig() {RPCResult::Type::STR, "address", "The value of the new multisig address."}, {RPCResult::Type::STR_HEX, "redeemScript", "The string value of the hex-encoded redemption script."}, {RPCResult::Type::STR, "descriptor", "The descriptor for this multisig"}, - {RPCResult::Type::ARR, "warnings", /* optional */ true, "Any warnings resulting from the creation of this multisig", + {RPCResult::Type::ARR, "warnings", /*optional=*/true, "Any warnings resulting from the creation of this multisig", { {RPCResult::Type::STR, "", ""}, }}, diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 1bde4fccbb..225feabf14 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -93,7 +93,7 @@ static RPCHelpMan getpeerinfo() { return RPCHelpMan{ "getpeerinfo", - "\nReturns data about each connected network node as a json array of objects.\n", + "Returns data about each connected network peer as a json array of objects.", {}, RPCResult{ RPCResult::Type::ARR, "", "", @@ -105,7 +105,7 @@ static RPCHelpMan getpeerinfo() {RPCResult::Type::STR, "addr", "(host:port) The IP address and port of the peer"}, {RPCResult::Type::STR, "addrbind", /*optional=*/true, "(ip:port) Bind address of the connection to the peer"}, {RPCResult::Type::STR, "addrlocal", /*optional=*/true, "(ip:port) Local address as reported by the peer"}, - {RPCResult::Type::STR, "network", "Network (" + Join(GetNetworkNames(/* append_unroutable */ true), ", ") + ")"}, + {RPCResult::Type::STR, "network", "Network (" + Join(GetNetworkNames(/*append_unroutable=*/true), ", ") + ")"}, {RPCResult::Type::NUM, "mapped_as", /*optional=*/true, "The AS in the BGP route to the peer used for diversifying\n" "peer selection (only available if the asmap config flag is set)"}, {RPCResult::Type::STR_HEX, "services", "The services offered"}, @@ -113,7 +113,7 @@ static RPCHelpMan getpeerinfo() { {RPCResult::Type::STR, "SERVICE_NAME", "the service name if it is recognised"} }}, - {RPCResult::Type::BOOL, "relaytxes", "Whether peer has asked us to relay transactions to it"}, + {RPCResult::Type::BOOL, "relaytxes", /*optional=*/true, "Whether peer has asked us to relay transactions to it"}, {RPCResult::Type::NUM_TIME, "lastsend", "The " + UNIX_EPOCH_TIME + " of the last send"}, {RPCResult::Type::NUM_TIME, "lastrecv", "The " + UNIX_EPOCH_TIME + " of the last receive"}, {RPCResult::Type::NUM_TIME, "last_transaction", "The " + UNIX_EPOCH_TIME + " of the last valid transaction received from this peer"}, @@ -144,7 +144,7 @@ static RPCHelpMan getpeerinfo() { {RPCResult::Type::STR, "permission_type", Join(NET_PERMISSIONS_DOC, ",\n") + ".\n"}, }}, - {RPCResult::Type::NUM, "minfeefilter", "The minimum fee rate for transactions this peer accepts"}, + {RPCResult::Type::NUM, "minfeefilter", /*optional=*/true, "The minimum fee rate for transactions this peer accepts"}, {RPCResult::Type::OBJ_DYN, "bytessent_per_msg", "", { {RPCResult::Type::NUM, "msg", "The total bytes sent aggregated by message type\n" @@ -197,7 +197,6 @@ static RPCHelpMan getpeerinfo() } obj.pushKV("services", strprintf("%016x", stats.nServices)); obj.pushKV("servicesnames", GetServicesNames(stats.nServices)); - obj.pushKV("relaytxes", stats.fRelayTxes); obj.pushKV("lastsend", count_seconds(stats.m_last_send)); obj.pushKV("lastrecv", count_seconds(stats.m_last_recv)); obj.pushKV("last_transaction", count_seconds(stats.m_last_tx_time)); @@ -232,6 +231,8 @@ static RPCHelpMan getpeerinfo() heights.push_back(height); } obj.pushKV("inflight", heights); + obj.pushKV("relaytxes", statestats.m_relay_txs); + obj.pushKV("minfeefilter", ValueFromAmount(statestats.m_fee_filter_received)); obj.pushKV("addr_relay_enabled", statestats.m_addr_relay_enabled); obj.pushKV("addr_processed", statestats.m_addr_processed); obj.pushKV("addr_rate_limited", statestats.m_addr_rate_limited); @@ -241,7 +242,6 @@ static RPCHelpMan getpeerinfo() permissions.push_back(permission); } obj.pushKV("permissions", permissions); - obj.pushKV("minfeefilter", ValueFromAmount(stats.minFeeFilter)); UniValue sendPerMsgCmd(UniValue::VOBJ); for (const auto& i : stats.mapSendBytesPerMsgCmd) { @@ -888,7 +888,7 @@ static RPCHelpMan getnodeaddresses() } // returns a shuffled list of CAddress - const std::vector<CAddress> vAddr{connman.GetAddresses(count, /* max_pct */ 0, network)}; + const std::vector<CAddress> vAddr{connman.GetAddresses(count, /*max_pct=*/0, network)}; UniValue ret(UniValue::VARR); for (const CAddress& addr : vAddr) { diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 6272a7c8cf..8e4b396da9 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -11,7 +11,6 @@ #include <core_io.h> #include <index/txindex.h> #include <key_io.h> -#include <merkleblock.h> #include <node/blockstorage.h> #include <node/coin.h> #include <node/context.h> @@ -34,9 +33,9 @@ #include <script/standard.h> #include <uint256.h> #include <util/bip32.h> -#include <util/moneystr.h> #include <util/strencodings.h> #include <util/string.h> +#include <util/vector.h> #include <validation.h> #include <validationinterface.h> @@ -47,7 +46,6 @@ using node::AnalyzePSBT; using node::BroadcastTransaction; -using node::DEFAULT_MAX_RAW_TX_FEE_RATE; using node::FindCoins; using node::GetTransaction; using node::NodeContext; @@ -61,13 +59,13 @@ static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& // Blockchain contextual information (confirmations and blocktime) is not // available to code in bitcoin-common, so we query them here and push the // data into the returned UniValue. - TxToUniv(tx, uint256(), entry, true, RPCSerializationFlags()); + TxToUniv(tx, /*block_hash=*/uint256(), entry, /*include_hex=*/true, RPCSerializationFlags()); if (!hashBlock.IsNull()) { LOCK(cs_main); entry.pushKV("blockhash", hashBlock.GetHex()); - CBlockIndex* pindex = active_chainstate.m_blockman.LookupBlockIndex(hashBlock); + const CBlockIndex* pindex = active_chainstate.m_blockman.LookupBlockIndex(hashBlock); if (pindex) { if (active_chainstate.m_chain.Contains(pindex)) { entry.pushKV("confirmations", 1 + active_chainstate.m_chain.Height() - pindex->nHeight); @@ -80,6 +78,54 @@ static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& } } +static std::vector<RPCResult> DecodeTxDoc(const std::string& txid_field_doc) +{ + return { + {RPCResult::Type::STR_HEX, "txid", txid_field_doc}, + {RPCResult::Type::STR_HEX, "hash", "The transaction hash (differs from txid for witness transactions)"}, + {RPCResult::Type::NUM, "size", "The serialized transaction size"}, + {RPCResult::Type::NUM, "vsize", "The virtual transaction size (differs from size for witness transactions)"}, + {RPCResult::Type::NUM, "weight", "The transaction's weight (between vsize*4-3 and vsize*4)"}, + {RPCResult::Type::NUM, "version", "The version"}, + {RPCResult::Type::NUM_TIME, "locktime", "The lock time"}, + {RPCResult::Type::ARR, "vin", "", + { + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR_HEX, "coinbase", /*optional=*/true, "The coinbase value (only if coinbase transaction)"}, + {RPCResult::Type::STR_HEX, "txid", /*optional=*/true, "The transaction id (if not coinbase transaction)"}, + {RPCResult::Type::NUM, "vout", /*optional=*/true, "The output number (if not coinbase transaction)"}, + {RPCResult::Type::OBJ, "scriptSig", /*optional=*/true, "The script (if not coinbase transaction)", + { + {RPCResult::Type::STR, "asm", "asm"}, + {RPCResult::Type::STR_HEX, "hex", "hex"}, + }}, + {RPCResult::Type::ARR, "txinwitness", /*optional=*/true, "", + { + {RPCResult::Type::STR_HEX, "hex", "hex-encoded witness data (if any)"}, + }}, + {RPCResult::Type::NUM, "sequence", "The script sequence number"}, + }}, + }}, + {RPCResult::Type::ARR, "vout", "", + { + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR_AMOUNT, "value", "The value in " + CURRENCY_UNIT}, + {RPCResult::Type::NUM, "n", "index"}, + {RPCResult::Type::OBJ, "scriptPubKey", "", + { + {RPCResult::Type::STR, "asm", "the asm"}, + {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, + {RPCResult::Type::STR_HEX, "hex", "the hex"}, + {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, + {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, + }}, + }}, + }}, + }; +} + static std::vector<RPCArg> CreateTxDoc() { return { @@ -121,7 +167,7 @@ static RPCHelpMan getrawtransaction() { return RPCHelpMan{ "getrawtransaction", - "\nReturn the raw transaction data.\n" + "Return the raw transaction data.\n" "\nBy default, this call only returns a transaction if it is in the mempool. If -txindex is enabled\n" "and no blockhash argument is passed, it will return the transaction if it is in the mempool or any block.\n" @@ -130,7 +176,7 @@ static RPCHelpMan getrawtransaction() "\nHint: Use gettransaction for wallet transactions.\n" "\nIf verbose is 'true', returns an Object with information about 'txid'.\n" - "If verbose is 'false' or omitted, returns a string that is serialized, hex-encoded data for 'txid'.\n", + "If verbose is 'false' or omitted, returns a string that is serialized, hex-encoded data for 'txid'.", { {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id"}, {"verbose", RPCArg::Type::BOOL, RPCArg::Default{false}, "If false, return a string, otherwise return a json object"}, @@ -142,55 +188,16 @@ static RPCHelpMan getrawtransaction() }, RPCResult{"if verbose is set to true", RPCResult::Type::OBJ, "", "", + Cat<std::vector<RPCResult>>( { {RPCResult::Type::BOOL, "in_active_chain", /*optional=*/true, "Whether specified block is in the active chain or not (only present with explicit \"blockhash\" argument)"}, - {RPCResult::Type::STR_HEX, "hex", "The serialized, hex-encoded data for 'txid'"}, - {RPCResult::Type::STR_HEX, "txid", "The transaction id (same as provided)"}, - {RPCResult::Type::STR_HEX, "hash", "The transaction hash (differs from txid for witness transactions)"}, - {RPCResult::Type::NUM, "size", "The serialized transaction size"}, - {RPCResult::Type::NUM, "vsize", "The virtual transaction size (differs from size for witness transactions)"}, - {RPCResult::Type::NUM, "weight", "The transaction's weight (between vsize*4-3 and vsize*4)"}, - {RPCResult::Type::NUM, "version", "The version"}, - {RPCResult::Type::NUM_TIME, "locktime", "The lock time"}, - {RPCResult::Type::ARR, "vin", "", - { - {RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR_HEX, "txid", "The transaction id"}, - {RPCResult::Type::NUM, "vout", "The output number"}, - {RPCResult::Type::OBJ, "scriptSig", "The script", - { - {RPCResult::Type::STR, "asm", "asm"}, - {RPCResult::Type::STR_HEX, "hex", "hex"}, - }}, - {RPCResult::Type::NUM, "sequence", "The script sequence number"}, - {RPCResult::Type::ARR, "txinwitness", /*optional=*/true, "", - { - {RPCResult::Type::STR_HEX, "hex", "hex-encoded witness data (if any)"}, - }}, - }}, - }}, - {RPCResult::Type::ARR, "vout", "", - { - {RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::NUM, "value", "The value in " + CURRENCY_UNIT}, - {RPCResult::Type::NUM, "n", "index"}, - {RPCResult::Type::OBJ, "scriptPubKey", "", - { - {RPCResult::Type::STR, "asm", "the asm"}, - {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, - {RPCResult::Type::STR, "hex", "the hex"}, - {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, - {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, - }}, - }}, - }}, {RPCResult::Type::STR_HEX, "blockhash", /*optional=*/true, "the block hash"}, {RPCResult::Type::NUM, "confirmations", /*optional=*/true, "The confirmations"}, {RPCResult::Type::NUM_TIME, "blocktime", /*optional=*/true, "The block time expressed in " + UNIX_EPOCH_TIME}, {RPCResult::Type::NUM, "time", /*optional=*/true, "Same as \"blocktime\""}, - } + {RPCResult::Type::STR_HEX, "hex", "The serialized, hex-encoded data for 'txid'"}, + }, + DecodeTxDoc(/*txid_field_doc=*/"The transaction id (same as provided)")), }, }, RPCExamples{ @@ -207,7 +214,7 @@ static RPCHelpMan getrawtransaction() bool in_active_chain = true; uint256 hash = ParseHashV(request.params[0], "parameter 1"); - CBlockIndex* blockindex = nullptr; + const CBlockIndex* blockindex = nullptr; if (hash == Params().GenesisBlock().hashMerkleRoot) { // Special exception for the genesis block coinbase transaction @@ -268,155 +275,6 @@ static RPCHelpMan getrawtransaction() }; } -static RPCHelpMan gettxoutproof() -{ - return RPCHelpMan{"gettxoutproof", - "\nReturns a hex-encoded proof that \"txid\" was included in a block.\n" - "\nNOTE: By default this function only works sometimes. This is when there is an\n" - "unspent output in the utxo for this transaction. To make it always work,\n" - "you need to maintain a transaction index, using the -txindex command line option or\n" - "specify the block in which the transaction is included manually (by blockhash).\n", - { - {"txids", RPCArg::Type::ARR, RPCArg::Optional::NO, "The txids to filter", - { - {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A transaction hash"}, - }, - }, - {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED_NAMED_ARG, "If specified, looks for txid in the block with this hash"}, - }, - RPCResult{ - RPCResult::Type::STR, "data", "A string that is a serialized, hex-encoded data for the proof." - }, - RPCExamples{""}, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - std::set<uint256> setTxids; - UniValue txids = request.params[0].get_array(); - if (txids.empty()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter 'txids' cannot be empty"); - } - for (unsigned int idx = 0; idx < txids.size(); idx++) { - auto ret = setTxids.insert(ParseHashV(txids[idx], "txid")); - if (!ret.second) { - throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid parameter, duplicated txid: ") + txids[idx].get_str()); - } - } - - CBlockIndex* pblockindex = nullptr; - uint256 hashBlock; - ChainstateManager& chainman = EnsureAnyChainman(request.context); - if (!request.params[1].isNull()) { - LOCK(cs_main); - hashBlock = ParseHashV(request.params[1], "blockhash"); - pblockindex = chainman.m_blockman.LookupBlockIndex(hashBlock); - if (!pblockindex) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); - } - } else { - LOCK(cs_main); - CChainState& active_chainstate = chainman.ActiveChainstate(); - - // Loop through txids and try to find which block they're in. Exit loop once a block is found. - for (const auto& tx : setTxids) { - const Coin& coin = AccessByTxid(active_chainstate.CoinsTip(), tx); - if (!coin.IsSpent()) { - pblockindex = active_chainstate.m_chain[coin.nHeight]; - break; - } - } - } - - - // Allow txindex to catch up if we need to query it and before we acquire cs_main. - if (g_txindex && !pblockindex) { - g_txindex->BlockUntilSyncedToCurrentChain(); - } - - LOCK(cs_main); - - if (pblockindex == nullptr) { - const CTransactionRef tx = GetTransaction(/* block_index */ nullptr, /* mempool */ nullptr, *setTxids.begin(), Params().GetConsensus(), hashBlock); - if (!tx || hashBlock.IsNull()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not yet in block"); - } - pblockindex = chainman.m_blockman.LookupBlockIndex(hashBlock); - if (!pblockindex) { - throw JSONRPCError(RPC_INTERNAL_ERROR, "Transaction index corrupt"); - } - } - - CBlock block; - if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) { - throw JSONRPCError(RPC_INTERNAL_ERROR, "Can't read block from disk"); - } - - unsigned int ntxFound = 0; - for (const auto& tx : block.vtx) { - if (setTxids.count(tx->GetHash())) { - ntxFound++; - } - } - if (ntxFound != setTxids.size()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Not all transactions found in specified or retrieved block"); - } - - CDataStream ssMB(SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS); - CMerkleBlock mb(block, setTxids); - ssMB << mb; - std::string strHex = HexStr(ssMB); - return strHex; -}, - }; -} - -static RPCHelpMan verifytxoutproof() -{ - return RPCHelpMan{"verifytxoutproof", - "\nVerifies that a proof points to a transaction in a block, returning the transaction it commits to\n" - "and throwing an RPC error if the block is not in our best chain\n", - { - {"proof", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex-encoded proof generated by gettxoutproof"}, - }, - RPCResult{ - RPCResult::Type::ARR, "", "", - { - {RPCResult::Type::STR_HEX, "txid", "The txid(s) which the proof commits to, or empty array if the proof cannot be validated."}, - } - }, - RPCExamples{""}, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - CDataStream ssMB(ParseHexV(request.params[0], "proof"), SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS); - CMerkleBlock merkleBlock; - ssMB >> merkleBlock; - - UniValue res(UniValue::VARR); - - std::vector<uint256> vMatch; - std::vector<unsigned int> vIndex; - if (merkleBlock.txn.ExtractMatches(vMatch, vIndex) != merkleBlock.header.hashMerkleRoot) - return res; - - ChainstateManager& chainman = EnsureAnyChainman(request.context); - LOCK(cs_main); - - const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(merkleBlock.header.GetHash()); - if (!pindex || !chainman.ActiveChain().Contains(pindex) || pindex->nTx == 0) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found in chain"); - } - - // Check if proof is valid, only add results if so - if (pindex->nTx == merkleBlock.txn.GetNumTransactions()) { - for (const uint256& hash : vMatch) { - res.push_back(hash.GetHex()); - } - } - - return res; -}, - }; -} - static RPCHelpMan createrawtransaction() { return RPCHelpMan{"createrawtransaction", @@ -459,7 +317,7 @@ static RPCHelpMan createrawtransaction() static RPCHelpMan decoderawtransaction() { return RPCHelpMan{"decoderawtransaction", - "\nReturn a JSON object representing the serialized, hex-encoded transaction.\n", + "Return a JSON object representing the serialized, hex-encoded transaction.", { {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction hex string"}, {"iswitness", RPCArg::Type::BOOL, RPCArg::DefaultHint{"depends on heuristic tests"}, "Whether the transaction hex is a serialized witness transaction.\n" @@ -472,50 +330,7 @@ static RPCHelpMan decoderawtransaction() }, RPCResult{ RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR_HEX, "txid", "The transaction id"}, - {RPCResult::Type::STR_HEX, "hash", "The transaction hash (differs from txid for witness transactions)"}, - {RPCResult::Type::NUM, "size", "The transaction size"}, - {RPCResult::Type::NUM, "vsize", "The virtual transaction size (differs from size for witness transactions)"}, - {RPCResult::Type::NUM, "weight", "The transaction's weight (between vsize*4 - 3 and vsize*4)"}, - {RPCResult::Type::NUM, "version", "The version"}, - {RPCResult::Type::NUM_TIME, "locktime", "The lock time"}, - {RPCResult::Type::ARR, "vin", "", - { - {RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR_HEX, "coinbase", /*optional=*/true, ""}, - {RPCResult::Type::STR_HEX, "txid", /*optional=*/true, "The transaction id"}, - {RPCResult::Type::NUM, "vout", /*optional=*/true, "The output number"}, - {RPCResult::Type::OBJ, "scriptSig", /*optional=*/true, "The script", - { - {RPCResult::Type::STR, "asm", "asm"}, - {RPCResult::Type::STR_HEX, "hex", "hex"}, - }}, - {RPCResult::Type::ARR, "txinwitness", /*optional=*/true, "", - { - {RPCResult::Type::STR_HEX, "hex", "hex-encoded witness data (if any)"}, - }}, - {RPCResult::Type::NUM, "sequence", "The script sequence number"}, - }}, - }}, - {RPCResult::Type::ARR, "vout", "", - { - {RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::NUM, "value", "The value in " + CURRENCY_UNIT}, - {RPCResult::Type::NUM, "n", "index"}, - {RPCResult::Type::OBJ, "scriptPubKey", "", - { - {RPCResult::Type::STR, "asm", "the asm"}, - {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, - {RPCResult::Type::STR_HEX, "hex", "the hex"}, - {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, - {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, - }}, - }}, - }}, - } + DecodeTxDoc(/*txid_field_doc=*/"The transaction id"), }, RPCExamples{ HelpExampleCli("decoderawtransaction", "\"hexstring\"") @@ -535,7 +350,7 @@ static RPCHelpMan decoderawtransaction() } UniValue result(UniValue::VOBJ); - TxToUniv(CTransaction(std::move(mtx)), uint256(), result, false); + TxToUniv(CTransaction(std::move(mtx)), /*block_hash=*/uint256(), /*entry=*/result, /*include_hex=*/false); return result; }, @@ -587,7 +402,7 @@ static RPCHelpMan decodescript() } else { // Empty scripts are valid } - ScriptPubKeyToUniv(script, r, /* include_hex */ false); + ScriptToUniv(script, /*out=*/r, /*include_hex=*/false, /*include_address=*/true); std::vector<std::vector<unsigned char>> solutions_data; const TxoutType which_type{Solver(script, solutions_data)}; @@ -664,7 +479,7 @@ static RPCHelpMan decodescript() // Scripts that are not fit for P2WPKH are encoded as P2WSH. segwitScr = GetScriptForDestination(WitnessV0ScriptHash(script)); } - ScriptPubKeyToUniv(segwitScr, sr, /* include_hex */ true); + ScriptToUniv(segwitScr, /*out=*/sr, /*include_hex=*/true, /*include_address=*/true); sr.pushKV("p2sh-segwit", EncodeDestination(ScriptHash(segwitScr))); r.pushKV("segwit", sr); } @@ -864,210 +679,6 @@ static RPCHelpMan signrawtransactionwithkey() }; } -static RPCHelpMan sendrawtransaction() -{ - return RPCHelpMan{"sendrawtransaction", - "\nSubmit a raw transaction (serialized, hex-encoded) to local node and network.\n" - "\nThe transaction will be sent unconditionally to all peers, so using sendrawtransaction\n" - "for manual rebroadcast may degrade privacy by leaking the transaction's origin, as\n" - "nodes will normally not rebroadcast non-wallet transactions already in their mempool.\n" - "\nA specific exception, RPC_TRANSACTION_ALREADY_IN_CHAIN, may throw if the transaction cannot be added to the mempool.\n" - "\nRelated RPCs: createrawtransaction, signrawtransactionwithkey\n", - { - {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex string of the raw transaction"}, - {"maxfeerate", RPCArg::Type::AMOUNT, RPCArg::Default{FormatMoney(DEFAULT_MAX_RAW_TX_FEE_RATE.GetFeePerK())}, - "Reject transactions whose fee rate is higher than the specified value, expressed in " + CURRENCY_UNIT + - "/kvB.\nSet to 0 to accept any fee rate.\n"}, - }, - RPCResult{ - RPCResult::Type::STR_HEX, "", "The transaction hash in hex" - }, - RPCExamples{ - "\nCreate a transaction\n" - + HelpExampleCli("createrawtransaction", "\"[{\\\"txid\\\" : \\\"mytxid\\\",\\\"vout\\\":0}]\" \"{\\\"myaddress\\\":0.01}\"") + - "Sign the transaction, and get back the hex\n" - + HelpExampleCli("signrawtransactionwithwallet", "\"myhex\"") + - "\nSend the transaction (signed hex)\n" - + HelpExampleCli("sendrawtransaction", "\"signedhex\"") + - "\nAs a JSON-RPC call\n" - + HelpExampleRpc("sendrawtransaction", "\"signedhex\"") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - RPCTypeCheck(request.params, { - UniValue::VSTR, - UniValueType(), // VNUM or VSTR, checked inside AmountFromValue() - }); - - CMutableTransaction mtx; - if (!DecodeHexTx(mtx, request.params[0].get_str())) { - throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed. Make sure the tx has at least one input."); - } - CTransactionRef tx(MakeTransactionRef(std::move(mtx))); - - const CFeeRate max_raw_tx_fee_rate = request.params[1].isNull() ? - DEFAULT_MAX_RAW_TX_FEE_RATE : - CFeeRate(AmountFromValue(request.params[1])); - - int64_t virtual_size = GetVirtualTransactionSize(*tx); - CAmount max_raw_tx_fee = max_raw_tx_fee_rate.GetFee(virtual_size); - - std::string err_string; - AssertLockNotHeld(cs_main); - NodeContext& node = EnsureAnyNodeContext(request.context); - const TransactionError err = BroadcastTransaction(node, tx, err_string, max_raw_tx_fee, /*relay*/ true, /*wait_callback*/ true); - if (TransactionError::OK != err) { - throw JSONRPCTransactionError(err, err_string); - } - - return tx->GetHash().GetHex(); -}, - }; -} - -static RPCHelpMan testmempoolaccept() -{ - return RPCHelpMan{"testmempoolaccept", - "\nReturns result of mempool acceptance tests indicating if raw transaction(s) (serialized, hex-encoded) would be accepted by mempool.\n" - "\nIf multiple transactions are passed in, parents must come before children and package policies apply: the transactions cannot conflict with any mempool transactions or each other.\n" - "\nIf one transaction fails, other transactions may not be fully validated (the 'allowed' key will be blank).\n" - "\nThe maximum number of transactions allowed is " + ToString(MAX_PACKAGE_COUNT) + ".\n" - "\nThis checks if transactions violate the consensus or policy rules.\n" - "\nSee sendrawtransaction call.\n", - { - {"rawtxs", RPCArg::Type::ARR, RPCArg::Optional::NO, "An array of hex strings of raw transactions.", - { - {"rawtx", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, - }, - }, - {"maxfeerate", RPCArg::Type::AMOUNT, RPCArg::Default{FormatMoney(DEFAULT_MAX_RAW_TX_FEE_RATE.GetFeePerK())}, - "Reject transactions whose fee rate is higher than the specified value, expressed in " + CURRENCY_UNIT + "/kvB\n"}, - }, - RPCResult{ - RPCResult::Type::ARR, "", "The result of the mempool acceptance test for each raw transaction in the input array.\n" - "Returns results for each transaction in the same order they were passed in.\n" - "Transactions that cannot be fully validated due to failures in other transactions will not contain an 'allowed' result.\n", - { - {RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR_HEX, "txid", "The transaction hash in hex"}, - {RPCResult::Type::STR_HEX, "wtxid", "The transaction witness hash in hex"}, - {RPCResult::Type::STR, "package-error", /*optional=*/true, "Package validation error, if any (only possible if rawtxs had more than 1 transaction)."}, - {RPCResult::Type::BOOL, "allowed", /*optional=*/true, "Whether this tx would be accepted to the mempool and pass client-specified maxfeerate. " - "If not present, the tx was not fully validated due to a failure in another tx in the list."}, - {RPCResult::Type::NUM, "vsize", /*optional=*/true, "Virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted (only present when 'allowed' is true)"}, - {RPCResult::Type::OBJ, "fees", /*optional=*/true, "Transaction fees (only present if 'allowed' is true)", - { - {RPCResult::Type::STR_AMOUNT, "base", "transaction fee in " + CURRENCY_UNIT}, - }}, - {RPCResult::Type::STR, "reject-reason", /*optional=*/true, "Rejection string (only present when 'allowed' is false)"}, - }}, - } - }, - RPCExamples{ - "\nCreate a transaction\n" - + HelpExampleCli("createrawtransaction", "\"[{\\\"txid\\\" : \\\"mytxid\\\",\\\"vout\\\":0}]\" \"{\\\"myaddress\\\":0.01}\"") + - "Sign the transaction, and get back the hex\n" - + HelpExampleCli("signrawtransactionwithwallet", "\"myhex\"") + - "\nTest acceptance of the transaction (signed hex)\n" - + HelpExampleCli("testmempoolaccept", R"('["signedhex"]')") + - "\nAs a JSON-RPC call\n" - + HelpExampleRpc("testmempoolaccept", "[\"signedhex\"]") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - RPCTypeCheck(request.params, { - UniValue::VARR, - UniValueType(), // VNUM or VSTR, checked inside AmountFromValue() - }); - const UniValue raw_transactions = request.params[0].get_array(); - if (raw_transactions.size() < 1 || raw_transactions.size() > MAX_PACKAGE_COUNT) { - throw JSONRPCError(RPC_INVALID_PARAMETER, - "Array must contain between 1 and " + ToString(MAX_PACKAGE_COUNT) + " transactions."); - } - - const CFeeRate max_raw_tx_fee_rate = request.params[1].isNull() ? - DEFAULT_MAX_RAW_TX_FEE_RATE : - CFeeRate(AmountFromValue(request.params[1])); - - std::vector<CTransactionRef> txns; - txns.reserve(raw_transactions.size()); - for (const auto& rawtx : raw_transactions.getValues()) { - CMutableTransaction mtx; - if (!DecodeHexTx(mtx, rawtx.get_str())) { - throw JSONRPCError(RPC_DESERIALIZATION_ERROR, - "TX decode failed: " + rawtx.get_str() + " Make sure the tx has at least one input."); - } - txns.emplace_back(MakeTransactionRef(std::move(mtx))); - } - - NodeContext& node = EnsureAnyNodeContext(request.context); - CTxMemPool& mempool = EnsureMemPool(node); - ChainstateManager& chainman = EnsureChainman(node); - CChainState& chainstate = chainman.ActiveChainstate(); - const PackageMempoolAcceptResult package_result = [&] { - LOCK(::cs_main); - if (txns.size() > 1) return ProcessNewPackage(chainstate, mempool, txns, /* test_accept */ true); - return PackageMempoolAcceptResult(txns[0]->GetWitnessHash(), - chainman.ProcessTransaction(txns[0], /*test_accept=*/ true)); - }(); - - UniValue rpc_result(UniValue::VARR); - // We will check transaction fees while we iterate through txns in order. If any transaction fee - // exceeds maxfeerate, we will leave the rest of the validation results blank, because it - // doesn't make sense to return a validation result for a transaction if its ancestor(s) would - // not be submitted. - bool exit_early{false}; - for (const auto& tx : txns) { - UniValue result_inner(UniValue::VOBJ); - result_inner.pushKV("txid", tx->GetHash().GetHex()); - result_inner.pushKV("wtxid", tx->GetWitnessHash().GetHex()); - if (package_result.m_state.GetResult() == PackageValidationResult::PCKG_POLICY) { - result_inner.pushKV("package-error", package_result.m_state.GetRejectReason()); - } - auto it = package_result.m_tx_results.find(tx->GetWitnessHash()); - if (exit_early || it == package_result.m_tx_results.end()) { - // Validation unfinished. Just return the txid and wtxid. - rpc_result.push_back(result_inner); - continue; - } - const auto& tx_result = it->second; - // Package testmempoolaccept doesn't allow transactions to already be in the mempool. - CHECK_NONFATAL(tx_result.m_result_type != MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); - if (tx_result.m_result_type == MempoolAcceptResult::ResultType::VALID) { - const CAmount fee = tx_result.m_base_fees.value(); - // Check that fee does not exceed maximum fee - const int64_t virtual_size = tx_result.m_vsize.value(); - const CAmount max_raw_tx_fee = max_raw_tx_fee_rate.GetFee(virtual_size); - if (max_raw_tx_fee && fee > max_raw_tx_fee) { - result_inner.pushKV("allowed", false); - result_inner.pushKV("reject-reason", "max-fee-exceeded"); - exit_early = true; - } else { - // Only return the fee and vsize if the transaction would pass ATMP. - // These can be used to calculate the feerate. - result_inner.pushKV("allowed", true); - result_inner.pushKV("vsize", virtual_size); - UniValue fees(UniValue::VOBJ); - fees.pushKV("base", ValueFromAmount(fee)); - result_inner.pushKV("fees", fees); - } - } else { - result_inner.pushKV("allowed", false); - const TxValidationState state = tx_result.m_state; - if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { - result_inner.pushKV("reject-reason", "missing-inputs"); - } else { - result_inner.pushKV("reject-reason", state.GetRejectReason()); - } - } - rpc_result.push_back(result_inner); - } - return rpc_result; -}, - }; -} - static RPCHelpMan decodepsbt() { return RPCHelpMan{ @@ -1121,6 +732,7 @@ static RPCHelpMan decodepsbt() {RPCResult::Type::OBJ, "scriptPubKey", "", { {RPCResult::Type::STR, "asm", "The asm"}, + {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, {RPCResult::Type::STR_HEX, "hex", "The hex"}, {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, @@ -1255,7 +867,7 @@ static RPCHelpMan decodepsbt() // Add the decoded tx UniValue tx_univ(UniValue::VOBJ); - TxToUniv(CTransaction(*psbtx.tx), uint256(), tx_univ, false); + TxToUniv(CTransaction(*psbtx.tx), /*block_hash=*/uint256(), /*entry=*/tx_univ, /*include_hex=*/false); result.pushKV("tx", tx_univ); // Add the global xpubs @@ -1311,7 +923,7 @@ static RPCHelpMan decodepsbt() txout = input.witness_utxo; UniValue o(UniValue::VOBJ); - ScriptPubKeyToUniv(txout.scriptPubKey, o, /* include_hex */ true); + ScriptToUniv(txout.scriptPubKey, /*out=*/o, /*include_hex=*/true, /*include_address=*/true); UniValue out(UniValue::VOBJ); out.pushKV("amount", ValueFromAmount(txout.nValue)); @@ -1325,7 +937,7 @@ static RPCHelpMan decodepsbt() txout = input.non_witness_utxo->vout[psbtx.tx->vin[i].prevout.n]; UniValue non_wit(UniValue::VOBJ); - TxToUniv(*input.non_witness_utxo, uint256(), non_wit, false); + TxToUniv(*input.non_witness_utxo, /*block_hash=*/uint256(), /*entry=*/non_wit, /*include_hex=*/false); in.pushKV("non_witness_utxo", non_wit); have_a_utxo = true; @@ -1358,12 +970,12 @@ static RPCHelpMan decodepsbt() // Redeem script and witness script if (!input.redeem_script.empty()) { UniValue r(UniValue::VOBJ); - ScriptToUniv(input.redeem_script, r); + ScriptToUniv(input.redeem_script, /*out=*/r); in.pushKV("redeem_script", r); } if (!input.witness_script.empty()) { UniValue r(UniValue::VOBJ); - ScriptToUniv(input.witness_script, r); + ScriptToUniv(input.witness_script, /*out=*/r); in.pushKV("witness_script", r); } @@ -1468,12 +1080,12 @@ static RPCHelpMan decodepsbt() // Redeem script and witness script if (!output.redeem_script.empty()) { UniValue r(UniValue::VOBJ); - ScriptToUniv(output.redeem_script, r); + ScriptToUniv(output.redeem_script, /*out=*/r); out.pushKV("redeem_script", r); } if (!output.witness_script.empty()) { UniValue r(UniValue::VOBJ); - ScriptToUniv(output.witness_script, r); + ScriptToUniv(output.witness_script, /*out=*/r); out.pushKV("witness_script", r); } @@ -2077,10 +1689,8 @@ static const CRPCCommand commands[] = { "rawtransactions", &createrawtransaction, }, { "rawtransactions", &decoderawtransaction, }, { "rawtransactions", &decodescript, }, - { "rawtransactions", &sendrawtransaction, }, { "rawtransactions", &combinerawtransaction, }, { "rawtransactions", &signrawtransactionwithkey, }, - { "rawtransactions", &testmempoolaccept, }, { "rawtransactions", &decodepsbt, }, { "rawtransactions", &combinepsbt, }, { "rawtransactions", &finalizepsbt, }, @@ -2089,9 +1699,6 @@ static const CRPCCommand commands[] = { "rawtransactions", &utxoupdatepsbt, }, { "rawtransactions", &joinpsbts, }, { "rawtransactions", &analyzepsbt, }, - - { "blockchain", &gettxoutproof, }, - { "blockchain", &verifytxoutproof, }, }; // clang-format on for (const auto& c : commands) { diff --git a/src/rpc/register.h b/src/rpc/register.h index c5055cc9d7..5a604ad428 100644 --- a/src/rpc/register.h +++ b/src/rpc/register.h @@ -9,22 +9,20 @@ * headers for everything under src/rpc/ */ class CRPCTable; -/** Register block chain RPC commands */ void RegisterBlockchainRPCCommands(CRPCTable &tableRPC); -/** Register P2P networking RPC commands */ +void RegisterMempoolRPCCommands(CRPCTable&); +void RegisterTxoutProofRPCCommands(CRPCTable&); void RegisterNetRPCCommands(CRPCTable &tableRPC); -/** Register miscellaneous RPC commands */ void RegisterMiscRPCCommands(CRPCTable &tableRPC); -/** Register mining RPC commands */ void RegisterMiningRPCCommands(CRPCTable &tableRPC); -/** Register raw transaction RPC commands */ void RegisterRawTransactionRPCCommands(CRPCTable &tableRPC); -/** Register raw transaction RPC commands */ void RegisterSignerRPCCommands(CRPCTable &tableRPC); static inline void RegisterAllCoreRPCCommands(CRPCTable &t) { RegisterBlockchainRPCCommands(t); + RegisterMempoolRPCCommands(t); + RegisterTxoutProofRPCCommands(t); RegisterNetRPCCommands(t); RegisterMiscRPCCommands(t); RegisterMiningRPCCommands(t); diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index c70236cc1c..333ed6f5da 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -167,7 +167,7 @@ static RPCHelpMan stop() // to the client (intended for testing) "\nRequest a graceful shutdown of " PACKAGE_NAME ".", { - {"wait", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "how long to wait in ms", "", {}, /* hidden */ true}, + {"wait", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "how long to wait in ms", "", {}, /*hidden=*/true}, }, RPCResult{RPCResult::Type::STR, "", "A string with the content '" + RESULT + "'"}, RPCExamples{""}, diff --git a/src/rpc/txoutproof.cpp b/src/rpc/txoutproof.cpp new file mode 100644 index 0000000000..a5443b0329 --- /dev/null +++ b/src/rpc/txoutproof.cpp @@ -0,0 +1,183 @@ +// Copyright (c) 2010 Satoshi Nakamoto +// Copyright (c) 2009-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chain.h> +#include <chainparams.h> +#include <coins.h> +#include <index/txindex.h> +#include <merkleblock.h> +#include <node/blockstorage.h> +#include <primitives/transaction.h> +#include <rpc/server.h> +#include <rpc/server_util.h> +#include <rpc/util.h> +#include <univalue.h> +#include <util/strencodings.h> +#include <validation.h> + +using node::GetTransaction; +using node::ReadBlockFromDisk; + +static RPCHelpMan gettxoutproof() +{ + return RPCHelpMan{"gettxoutproof", + "\nReturns a hex-encoded proof that \"txid\" was included in a block.\n" + "\nNOTE: By default this function only works sometimes. This is when there is an\n" + "unspent output in the utxo for this transaction. To make it always work,\n" + "you need to maintain a transaction index, using the -txindex command line option or\n" + "specify the block in which the transaction is included manually (by blockhash).\n", + { + {"txids", RPCArg::Type::ARR, RPCArg::Optional::NO, "The txids to filter", + { + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A transaction hash"}, + }, + }, + {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED_NAMED_ARG, "If specified, looks for txid in the block with this hash"}, + }, + RPCResult{ + RPCResult::Type::STR, "data", "A string that is a serialized, hex-encoded data for the proof." + }, + RPCExamples{""}, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + std::set<uint256> setTxids; + UniValue txids = request.params[0].get_array(); + if (txids.empty()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter 'txids' cannot be empty"); + } + for (unsigned int idx = 0; idx < txids.size(); idx++) { + auto ret = setTxids.insert(ParseHashV(txids[idx], "txid")); + if (!ret.second) { + throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid parameter, duplicated txid: ") + txids[idx].get_str()); + } + } + + const CBlockIndex* pblockindex = nullptr; + uint256 hashBlock; + ChainstateManager& chainman = EnsureAnyChainman(request.context); + if (!request.params[1].isNull()) { + LOCK(cs_main); + hashBlock = ParseHashV(request.params[1], "blockhash"); + pblockindex = chainman.m_blockman.LookupBlockIndex(hashBlock); + if (!pblockindex) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + } + } else { + LOCK(cs_main); + CChainState& active_chainstate = chainman.ActiveChainstate(); + + // Loop through txids and try to find which block they're in. Exit loop once a block is found. + for (const auto& tx : setTxids) { + const Coin& coin = AccessByTxid(active_chainstate.CoinsTip(), tx); + if (!coin.IsSpent()) { + pblockindex = active_chainstate.m_chain[coin.nHeight]; + break; + } + } + } + + + // Allow txindex to catch up if we need to query it and before we acquire cs_main. + if (g_txindex && !pblockindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + + LOCK(cs_main); + + if (pblockindex == nullptr) { + const CTransactionRef tx = GetTransaction(/*block_index=*/nullptr, /*mempool=*/nullptr, *setTxids.begin(), Params().GetConsensus(), hashBlock); + if (!tx || hashBlock.IsNull()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not yet in block"); + } + pblockindex = chainman.m_blockman.LookupBlockIndex(hashBlock); + if (!pblockindex) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Transaction index corrupt"); + } + } + + CBlock block; + if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Can't read block from disk"); + } + + unsigned int ntxFound = 0; + for (const auto& tx : block.vtx) { + if (setTxids.count(tx->GetHash())) { + ntxFound++; + } + } + if (ntxFound != setTxids.size()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Not all transactions found in specified or retrieved block"); + } + + CDataStream ssMB(SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS); + CMerkleBlock mb(block, setTxids); + ssMB << mb; + std::string strHex = HexStr(ssMB); + return strHex; + }, + }; +} + +static RPCHelpMan verifytxoutproof() +{ + return RPCHelpMan{"verifytxoutproof", + "\nVerifies that a proof points to a transaction in a block, returning the transaction it commits to\n" + "and throwing an RPC error if the block is not in our best chain\n", + { + {"proof", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex-encoded proof generated by gettxoutproof"}, + }, + RPCResult{ + RPCResult::Type::ARR, "", "", + { + {RPCResult::Type::STR_HEX, "txid", "The txid(s) which the proof commits to, or empty array if the proof cannot be validated."}, + } + }, + RPCExamples{""}, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + CDataStream ssMB(ParseHexV(request.params[0], "proof"), SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS); + CMerkleBlock merkleBlock; + ssMB >> merkleBlock; + + UniValue res(UniValue::VARR); + + std::vector<uint256> vMatch; + std::vector<unsigned int> vIndex; + if (merkleBlock.txn.ExtractMatches(vMatch, vIndex) != merkleBlock.header.hashMerkleRoot) + return res; + + ChainstateManager& chainman = EnsureAnyChainman(request.context); + LOCK(cs_main); + + const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(merkleBlock.header.GetHash()); + if (!pindex || !chainman.ActiveChain().Contains(pindex) || pindex->nTx == 0) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found in chain"); + } + + // Check if proof is valid, only add results if so + if (pindex->nTx == merkleBlock.txn.GetNumTransactions()) { + for (const uint256& hash : vMatch) { + res.push_back(hash.GetHex()); + } + } + + return res; + }, + }; +} + +void RegisterTxoutProofRPCCommands(CRPCTable& t) +{ + static const CRPCCommand commands[]{ + // category actor (function) + // -------- ---------------- + {"blockchain", &gettxoutproof}, + {"blockchain", &verifytxoutproof}, + }; + for (const auto& c : commands) { + t.appendCommand(c.name, &c); + } +} diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 7c859268be..9c9e6e9f11 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -774,7 +774,7 @@ void RPCResult::ToSections(Sections& sections, const OuterType outer_type, const // Elements in a JSON structure (dictionary or array) are separated by a comma const std::string maybe_separator{outer_type != OuterType::NONE ? "," : ""}; - // The key name if recursed into an dictionary + // The key name if recursed into a dictionary const std::string maybe_key{ outer_type == OuterType::OBJ ? "\"" + this->m_key_name + "\" : " : @@ -865,10 +865,11 @@ void RPCResult::ToSections(Sections& sections, const OuterType outer_type, const bool RPCResult::MatchesType(const UniValue& result) const { - switch (m_type) { - case Type::ELISION: { - return false; + if (m_skip_type_check) { + return true; } + switch (m_type) { + case Type::ELISION: case Type::ANY: { return true; } @@ -889,11 +890,52 @@ bool RPCResult::MatchesType(const UniValue& result) const } case Type::ARR_FIXED: case Type::ARR: { - return UniValue::VARR == result.getType(); + if (UniValue::VARR != result.getType()) return false; + for (size_t i{0}; i < result.get_array().size(); ++i) { + // If there are more results than documented, re-use the last doc_inner. + const RPCResult& doc_inner{m_inner.at(std::min(m_inner.size() - 1, i))}; + if (!doc_inner.MatchesType(result.get_array()[i])) return false; + } + return true; // empty result array is valid } case Type::OBJ_DYN: case Type::OBJ: { - return UniValue::VOBJ == result.getType(); + if (UniValue::VOBJ != result.getType()) return false; + if (!m_inner.empty() && m_inner.at(0).m_type == Type::ELISION) return true; + if (m_type == Type::OBJ_DYN) { + const RPCResult& doc_inner{m_inner.at(0)}; // Assume all types are the same, randomly pick the first + for (size_t i{0}; i < result.get_obj().size(); ++i) { + if (!doc_inner.MatchesType(result.get_obj()[i])) { + return false; + } + } + return true; // empty result obj is valid + } + std::set<std::string> doc_keys; + for (const auto& doc_entry : m_inner) { + doc_keys.insert(doc_entry.m_key_name); + } + std::map<std::string, UniValue> result_obj; + result.getObjMap(result_obj); + for (const auto& result_entry : result_obj) { + if (doc_keys.find(result_entry.first) == doc_keys.end()) { + return false; // missing documentation + } + } + + for (const auto& doc_entry : m_inner) { + const auto result_it{result_obj.find(doc_entry.m_key_name)}; + if (result_it == result_obj.end()) { + if (!doc_entry.m_optional) { + return false; // result is missing a required key + } + continue; + } + if (!doc_entry.MatchesType(result_it->second)) { + return false; // wrong type + } + } + return true; } } // no default case, so the compiler can warn about missing cases CHECK_NONFATAL(false); diff --git a/src/rpc/util.h b/src/rpc/util.h index 89d32d4193..e16fed75bc 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -256,6 +256,7 @@ struct RPCResult { const std::string m_key_name; //!< Only used for dicts const std::vector<RPCResult> m_inner; //!< Only used for arrays or dicts const bool m_optional; + const bool m_skip_type_check; const std::string m_description; const std::string m_cond; @@ -270,6 +271,7 @@ struct RPCResult { m_key_name{std::move(m_key_name)}, m_inner{std::move(inner)}, m_optional{optional}, + m_skip_type_check{false}, m_description{std::move(description)}, m_cond{std::move(cond)} { @@ -290,11 +292,13 @@ struct RPCResult { const std::string m_key_name, const bool optional, const std::string description, - const std::vector<RPCResult> inner = {}) + const std::vector<RPCResult> inner = {}, + bool skip_type_check = false) : m_type{std::move(type)}, m_key_name{std::move(m_key_name)}, m_inner{std::move(inner)}, m_optional{optional}, + m_skip_type_check{skip_type_check}, m_description{std::move(description)}, m_cond{} { @@ -305,8 +309,9 @@ struct RPCResult { const Type type, const std::string m_key_name, const std::string description, - const std::vector<RPCResult> inner = {}) - : RPCResult{type, m_key_name, false, description, inner} {} + const std::vector<RPCResult> inner = {}, + bool skip_type_check = false) + : RPCResult{type, m_key_name, false, description, inner, skip_type_check} {} /** Append the sections of the result. */ void ToSections(Sections& sections, OuterType outer_type = OuterType::NONE, const int current_indent = 0) const; diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 0b2ad3c553..197d009f7c 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -111,7 +111,7 @@ static void Repeat(CScheduler& s, CScheduler::Function f, std::chrono::milliseco void CScheduler::scheduleEvery(CScheduler::Function f, std::chrono::milliseconds delta) { - scheduleFromNow([=] { Repeat(*this, f, delta); }, delta); + scheduleFromNow([this, f, delta] { Repeat(*this, f, delta); }, delta); } size_t CScheduler::getQueueInfo(std::chrono::system_clock::time_point& first, diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 23540f6aef..cece0b60ce 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -1066,13 +1066,19 @@ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const auto expr = Expr(sp); if (Func("pk", expr)) { auto pubkey = ParsePubkey(key_exp_index, expr, ctx, out, error); - if (!pubkey) return nullptr; + if (!pubkey) { + error = strprintf("pk(): %s", error); + return nullptr; + } ++key_exp_index; return std::make_unique<PKDescriptor>(std::move(pubkey), ctx == ParseScriptContext::P2TR); } if ((ctx == ParseScriptContext::TOP || ctx == ParseScriptContext::P2SH || ctx == ParseScriptContext::P2WSH) && Func("pkh", expr)) { auto pubkey = ParsePubkey(key_exp_index, expr, ctx, out, error); - if (!pubkey) return nullptr; + if (!pubkey) { + error = strprintf("pkh(): %s", error); + return nullptr; + } ++key_exp_index; return std::make_unique<PKHDescriptor>(std::move(pubkey)); } else if (Func("pkh", expr)) { @@ -1081,7 +1087,10 @@ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const } if (ctx == ParseScriptContext::TOP && Func("combo", expr)) { auto pubkey = ParsePubkey(key_exp_index, expr, ctx, out, error); - if (!pubkey) return nullptr; + if (!pubkey) { + error = strprintf("combo(): %s", error); + return nullptr; + } ++key_exp_index; return std::make_unique<ComboDescriptor>(std::move(pubkey)); } else if (Func("combo", expr)) { @@ -1109,7 +1118,10 @@ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const } auto arg = Expr(expr); auto pk = ParsePubkey(key_exp_index, arg, ctx, out, error); - if (!pk) return nullptr; + if (!pk) { + error = strprintf("Multi: %s", error); + return nullptr; + } script_size += pk->GetSize() + 1; providers.emplace_back(std::move(pk)); key_exp_index++; @@ -1154,7 +1166,10 @@ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const } if ((ctx == ParseScriptContext::TOP || ctx == ParseScriptContext::P2SH) && Func("wpkh", expr)) { auto pubkey = ParsePubkey(key_exp_index, expr, ParseScriptContext::P2WPKH, out, error); - if (!pubkey) return nullptr; + if (!pubkey) { + error = strprintf("wpkh(): %s", error); + return nullptr; + } key_exp_index++; return std::make_unique<WPKHDescriptor>(std::move(pubkey)); } else if (Func("wpkh", expr)) { @@ -1191,7 +1206,10 @@ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const if (ctx == ParseScriptContext::TOP && Func("tr", expr)) { auto arg = Expr(expr); auto internal_key = ParsePubkey(key_exp_index, arg, ParseScriptContext::P2TR, out, error); - if (!internal_key) return nullptr; + if (!internal_key) { + error = strprintf("tr(): %s", error); + return nullptr; + } ++key_exp_index; std::vector<std::unique_ptr<DescriptorImpl>> subscripts; //!< list of script subexpressions std::vector<int> depths; //!< depth in the tree of each subexpression (same length subscripts) diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 11b1a1c887..c4d13d7283 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -225,31 +225,6 @@ bool static CheckPubKeyEncoding(const valtype &vchPubKey, unsigned int flags, co return true; } -bool CheckMinimalPush(const valtype& data, opcodetype opcode) { - // Excludes OP_1NEGATE, OP_1-16 since they are by definition minimal - assert(0 <= opcode && opcode <= OP_PUSHDATA4); - if (data.size() == 0) { - // Should have used OP_0. - return opcode == OP_0; - } else if (data.size() == 1 && data[0] >= 1 && data[0] <= 16) { - // Should have used OP_1 .. OP_16. - return false; - } else if (data.size() == 1 && data[0] == 0x81) { - // Should have used OP_1NEGATE. - return false; - } else if (data.size() <= 75) { - // Must have used a direct push (opcode indicating number of bytes pushed + those bytes). - return opcode == data.size(); - } else if (data.size() <= 255) { - // Must have used OP_PUSHDATA. - return opcode == OP_PUSHDATA1; - } else if (data.size() <= 65535) { - // Must have used OP_PUSHDATA2. - return opcode == OP_PUSHDATA2; - } - return true; -} - int FindAndDelete(CScript& script, const CScript& b) { int nFound = 0; @@ -2009,7 +1984,7 @@ bool VerifyScript(const CScript& scriptSig, const CScript& scriptPubKey, const C // The scriptSig must be _exactly_ CScript(), otherwise we reintroduce malleability. return set_error(serror, SCRIPT_ERR_WITNESS_MALLEATED); } - if (!VerifyWitnessProgram(*witness, witnessversion, witnessprogram, flags, checker, serror, /* is_p2sh */ false)) { + if (!VerifyWitnessProgram(*witness, witnessversion, witnessprogram, flags, checker, serror, /*is_p2sh=*/false)) { return false; } // Bypass the cleanstack check at the end. The actual stack is obviously not clean @@ -2054,7 +2029,7 @@ bool VerifyScript(const CScript& scriptSig, const CScript& scriptPubKey, const C // reintroduce malleability. return set_error(serror, SCRIPT_ERR_WITNESS_MALLEATED_P2SH); } - if (!VerifyWitnessProgram(*witness, witnessversion, witnessprogram, flags, checker, serror, /* is_p2sh */ true)) { + if (!VerifyWitnessProgram(*witness, witnessversion, witnessprogram, flags, checker, serror, /*is_p2sh=*/true)) { return false; } // Bypass the cleanstack check at the end. The actual stack is obviously not clean diff --git a/src/script/interpreter.h b/src/script/interpreter.h index cf1953ad22..fbd904fb7b 100644 --- a/src/script/interpreter.h +++ b/src/script/interpreter.h @@ -344,8 +344,6 @@ bool VerifyScript(const CScript& scriptSig, const CScript& scriptPubKey, const C size_t CountWitnessSigOps(const CScript& scriptSig, const CScript& scriptPubKey, const CScriptWitness* witness, unsigned int flags); -bool CheckMinimalPush(const std::vector<unsigned char>& data, opcodetype opcode); - int FindAndDelete(CScript& script, const CScript& b); #endif // BITCOIN_SCRIPT_INTERPRETER_H diff --git a/src/script/miniscript.cpp b/src/script/miniscript.cpp new file mode 100644 index 0000000000..d0bb937885 --- /dev/null +++ b/src/script/miniscript.cpp @@ -0,0 +1,348 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <string> +#include <vector> +#include <script/script.h> +#include <script/standard.h> +#include <script/miniscript.h> + +#include <assert.h> + +namespace miniscript { +namespace internal { + +Type SanitizeType(Type e) { + int num_types = (e << "K"_mst) + (e << "V"_mst) + (e << "B"_mst) + (e << "W"_mst); + if (num_types == 0) return ""_mst; // No valid type, don't care about the rest + assert(num_types == 1); // K, V, B, W all conflict with each other + bool ok = // Work around a GCC 4.8 bug that breaks user-defined literals in macro calls. + (!(e << "z"_mst) || !(e << "o"_mst)) && // z conflicts with o + (!(e << "n"_mst) || !(e << "z"_mst)) && // n conflicts with z + (!(e << "n"_mst) || !(e << "W"_mst)) && // n conflicts with W + (!(e << "V"_mst) || !(e << "d"_mst)) && // V conflicts with d + (!(e << "K"_mst) || (e << "u"_mst)) && // K implies u + (!(e << "V"_mst) || !(e << "u"_mst)) && // V conflicts with u + (!(e << "e"_mst) || !(e << "f"_mst)) && // e conflicts with f + (!(e << "e"_mst) || (e << "d"_mst)) && // e implies d + (!(e << "V"_mst) || !(e << "e"_mst)) && // V conflicts with e + (!(e << "d"_mst) || !(e << "f"_mst)) && // d conflicts with f + (!(e << "V"_mst) || (e << "f"_mst)) && // V implies f + (!(e << "K"_mst) || (e << "s"_mst)) && // K implies s + (!(e << "z"_mst) || (e << "m"_mst)); // z implies m + assert(ok); + return e; +} + +Type ComputeType(Fragment nodetype, Type x, Type y, Type z, const std::vector<Type>& sub_types, uint32_t k, size_t data_size, size_t n_subs, size_t n_keys) { + // Sanity check on data + if (nodetype == Fragment::SHA256 || nodetype == Fragment::HASH256) { + assert(data_size == 32); + } else if (nodetype == Fragment::RIPEMD160 || nodetype == Fragment::HASH160) { + assert(data_size == 20); + } else { + assert(data_size == 0); + } + // Sanity check on k + if (nodetype == Fragment::OLDER || nodetype == Fragment::AFTER) { + assert(k >= 1 && k < 0x80000000UL); + } else if (nodetype == Fragment::MULTI) { + assert(k >= 1 && k <= n_keys); + } else if (nodetype == Fragment::THRESH) { + assert(k >= 1 && k <= n_subs); + } else { + assert(k == 0); + } + // Sanity check on subs + if (nodetype == Fragment::AND_V || nodetype == Fragment::AND_B || nodetype == Fragment::OR_B || + nodetype == Fragment::OR_C || nodetype == Fragment::OR_I || nodetype == Fragment::OR_D) { + assert(n_subs == 2); + } else if (nodetype == Fragment::ANDOR) { + assert(n_subs == 3); + } else if (nodetype == Fragment::WRAP_A || nodetype == Fragment::WRAP_S || nodetype == Fragment::WRAP_C || + nodetype == Fragment::WRAP_D || nodetype == Fragment::WRAP_V || nodetype == Fragment::WRAP_J || + nodetype == Fragment::WRAP_N) { + assert(n_subs == 1); + } else if (nodetype != Fragment::THRESH) { + assert(n_subs == 0); + } + // Sanity check on keys + if (nodetype == Fragment::PK_K || nodetype == Fragment::PK_H) { + assert(n_keys == 1); + } else if (nodetype == Fragment::MULTI) { + assert(n_keys >= 1 && n_keys <= 20); + } else { + assert(n_keys == 0); + } + + // Below is the per-nodetype logic for computing the expression types. + // It heavily relies on Type's << operator (where "X << a_mst" means + // "X has all properties listed in a"). + switch (nodetype) { + case Fragment::PK_K: return "Konudemsxk"_mst; + case Fragment::PK_H: return "Knudemsxk"_mst; + case Fragment::OLDER: return + "g"_mst.If(k & CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG) | + "h"_mst.If(!(k & CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG)) | + "Bzfmxk"_mst; + case Fragment::AFTER: return + "i"_mst.If(k >= LOCKTIME_THRESHOLD) | + "j"_mst.If(k < LOCKTIME_THRESHOLD) | + "Bzfmxk"_mst; + case Fragment::SHA256: return "Bonudmk"_mst; + case Fragment::RIPEMD160: return "Bonudmk"_mst; + case Fragment::HASH256: return "Bonudmk"_mst; + case Fragment::HASH160: return "Bonudmk"_mst; + case Fragment::JUST_1: return "Bzufmxk"_mst; + case Fragment::JUST_0: return "Bzudemsxk"_mst; + case Fragment::WRAP_A: return + "W"_mst.If(x << "B"_mst) | // W=B_x + (x & "ghijk"_mst) | // g=g_x, h=h_x, i=i_x, j=j_x, k=k_x + (x & "udfems"_mst) | // u=u_x, d=d_x, f=f_x, e=e_x, m=m_x, s=s_x + "x"_mst; // x + case Fragment::WRAP_S: return + "W"_mst.If(x << "Bo"_mst) | // W=B_x*o_x + (x & "ghijk"_mst) | // g=g_x, h=h_x, i=i_x, j=j_x, k=k_x + (x & "udfemsx"_mst); // u=u_x, d=d_x, f=f_x, e=e_x, m=m_x, s=s_x, x=x_x + case Fragment::WRAP_C: return + "B"_mst.If(x << "K"_mst) | // B=K_x + (x & "ghijk"_mst) | // g=g_x, h=h_x, i=i_x, j=j_x, k=k_x + (x & "ondfem"_mst) | // o=o_x, n=n_x, d=d_x, f=f_x, e=e_x, m=m_x + "us"_mst; // u, s + case Fragment::WRAP_D: return + "B"_mst.If(x << "Vz"_mst) | // B=V_x*z_x + "o"_mst.If(x << "z"_mst) | // o=z_x + "e"_mst.If(x << "f"_mst) | // e=f_x + (x & "ghijk"_mst) | // g=g_x, h=h_x, i=i_x, j=j_x, k=k_x + (x & "ms"_mst) | // m=m_x, s=s_x + "nudx"_mst; // n, u, d, x + case Fragment::WRAP_V: return + "V"_mst.If(x << "B"_mst) | // V=B_x + (x & "ghijk"_mst) | // g=g_x, h=h_x, i=i_x, j=j_x, k=k_x + (x & "zonms"_mst) | // z=z_x, o=o_x, n=n_x, m=m_x, s=s_x + "fx"_mst; // f, x + case Fragment::WRAP_J: return + "B"_mst.If(x << "Bn"_mst) | // B=B_x*n_x + "e"_mst.If(x << "f"_mst) | // e=f_x + (x & "ghijk"_mst) | // g=g_x, h=h_x, i=i_x, j=j_x, k=k_x + (x & "oums"_mst) | // o=o_x, u=u_x, m=m_x, s=s_x + "ndx"_mst; // n, d, x + case Fragment::WRAP_N: return + (x & "ghijk"_mst) | // g=g_x, h=h_x, i=i_x, j=j_x, k=k_x + (x & "Bzondfems"_mst) | // B=B_x, z=z_x, o=o_x, n=n_x, d=d_x, f=f_x, e=e_x, m=m_x, s=s_x + "ux"_mst; // u, x + case Fragment::AND_V: return + (y & "KVB"_mst).If(x << "V"_mst) | // B=V_x*B_y, V=V_x*V_y, K=V_x*K_y + (x & "n"_mst) | (y & "n"_mst).If(x << "z"_mst) | // n=n_x+z_x*n_y + ((x | y) & "o"_mst).If((x | y) << "z"_mst) | // o=o_x*z_y+z_x*o_y + (x & y & "dmz"_mst) | // d=d_x*d_y, m=m_x*m_y, z=z_x*z_y + ((x | y) & "s"_mst) | // s=s_x+s_y + "f"_mst.If((y << "f"_mst) || (x << "s"_mst)) | // f=f_y+s_x + (y & "ux"_mst) | // u=u_y, x=x_y + ((x | y) & "ghij"_mst) | // g=g_x+g_y, h=h_x+h_y, i=i_x+i_y, j=j_x+j_y + "k"_mst.If(((x & y) << "k"_mst) && + !(((x << "g"_mst) && (y << "h"_mst)) || + ((x << "h"_mst) && (y << "g"_mst)) || + ((x << "i"_mst) && (y << "j"_mst)) || + ((x << "j"_mst) && (y << "i"_mst)))); // k=k_x*k_y*!(g_x*h_y + h_x*g_y + i_x*j_y + j_x*i_y) + case Fragment::AND_B: return + (x & "B"_mst).If(y << "W"_mst) | // B=B_x*W_y + ((x | y) & "o"_mst).If((x | y) << "z"_mst) | // o=o_x*z_y+z_x*o_y + (x & "n"_mst) | (y & "n"_mst).If(x << "z"_mst) | // n=n_x+z_x*n_y + (x & y & "e"_mst).If((x & y) << "s"_mst) | // e=e_x*e_y*s_x*s_y + (x & y & "dzm"_mst) | // d=d_x*d_y, z=z_x*z_y, m=m_x*m_y + "f"_mst.If(((x & y) << "f"_mst) || (x << "sf"_mst) || (y << "sf"_mst)) | // f=f_x*f_y + f_x*s_x + f_y*s_y + ((x | y) & "s"_mst) | // s=s_x+s_y + "ux"_mst | // u, x + ((x | y) & "ghij"_mst) | // g=g_x+g_y, h=h_x+h_y, i=i_x+i_y, j=j_x+j_y + "k"_mst.If(((x & y) << "k"_mst) && + !(((x << "g"_mst) && (y << "h"_mst)) || + ((x << "h"_mst) && (y << "g"_mst)) || + ((x << "i"_mst) && (y << "j"_mst)) || + ((x << "j"_mst) && (y << "i"_mst)))); // k=k_x*k_y*!(g_x*h_y + h_x*g_y + i_x*j_y + j_x*i_y) + case Fragment::OR_B: return + "B"_mst.If(x << "Bd"_mst && y << "Wd"_mst) | // B=B_x*d_x*W_x*d_y + ((x | y) & "o"_mst).If((x | y) << "z"_mst) | // o=o_x*z_y+z_x*o_y + (x & y & "m"_mst).If((x | y) << "s"_mst && (x & y) << "e"_mst) | // m=m_x*m_y*e_x*e_y*(s_x+s_y) + (x & y & "zse"_mst) | // z=z_x*z_y, s=s_x*s_y, e=e_x*e_y + "dux"_mst | // d, u, x + ((x | y) & "ghij"_mst) | // g=g_x+g_y, h=h_x+h_y, i=i_x+i_y, j=j_x+j_y + (x & y & "k"_mst); // k=k_x*k_y + case Fragment::OR_D: return + (y & "B"_mst).If(x << "Bdu"_mst) | // B=B_y*B_x*d_x*u_x + (x & "o"_mst).If(y << "z"_mst) | // o=o_x*z_y + (x & y & "m"_mst).If(x << "e"_mst && (x | y) << "s"_mst) | // m=m_x*m_y*e_x*(s_x+s_y) + (x & y & "zes"_mst) | // z=z_x*z_y, e=e_x*e_y, s=s_x*s_y + (y & "ufd"_mst) | // u=u_y, f=f_y, d=d_y + "x"_mst | // x + ((x | y) & "ghij"_mst) | // g=g_x+g_y, h=h_x+h_y, i=i_x+i_y, j=j_x+j_y + (x & y & "k"_mst); // k=k_x*k_y + case Fragment::OR_C: return + (y & "V"_mst).If(x << "Bdu"_mst) | // V=V_y*B_x*u_x*d_x + (x & "o"_mst).If(y << "z"_mst) | // o=o_x*z_y + (x & y & "m"_mst).If(x << "e"_mst && (x | y) << "s"_mst) | // m=m_x*m_y*e_x*(s_x+s_y) + (x & y & "zs"_mst) | // z=z_x*z_y, s=s_x*s_y + "fx"_mst | // f, x + ((x | y) & "ghij"_mst) | // g=g_x+g_y, h=h_x+h_y, i=i_x+i_y, j=j_x+j_y + (x & y & "k"_mst); // k=k_x*k_y + case Fragment::OR_I: return + (x & y & "VBKufs"_mst) | // V=V_x*V_y, B=B_x*B_y, K=K_x*K_y, u=u_x*u_y, f=f_x*f_y, s=s_x*s_y + "o"_mst.If((x & y) << "z"_mst) | // o=z_x*z_y + ((x | y) & "e"_mst).If((x | y) << "f"_mst) | // e=e_x*f_y+f_x*e_y + (x & y & "m"_mst).If((x | y) << "s"_mst) | // m=m_x*m_y*(s_x+s_y) + ((x | y) & "d"_mst) | // d=d_x+d_y + "x"_mst | // x + ((x | y) & "ghij"_mst) | // g=g_x+g_y, h=h_x+h_y, i=i_x+i_y, j=j_x+j_y + (x & y & "k"_mst); // k=k_x*k_y + case Fragment::ANDOR: return + (y & z & "BKV"_mst).If(x << "Bdu"_mst) | // B=B_x*d_x*u_x*B_y*B_z, K=B_x*d_x*u_x*K_y*K_z, V=B_x*d_x*u_x*V_y*V_z + (x & y & z & "z"_mst) | // z=z_x*z_y*z_z + ((x | (y & z)) & "o"_mst).If((x | (y & z)) << "z"_mst) | // o=o_x*z_y*z_z+z_x*o_y*o_z + (y & z & "u"_mst) | // u=u_y*u_z + (z & "f"_mst).If((x << "s"_mst) || (y << "f"_mst)) | // f=(s_x+f_y)*f_z + (z & "d"_mst) | // d=d_z + (x & z & "e"_mst).If(x << "s"_mst || y << "f"_mst) | // e=e_x*e_z*(s_x+f_y) + (x & y & z & "m"_mst).If(x << "e"_mst && (x | y | z) << "s"_mst) | // m=m_x*m_y*m_z*e_x*(s_x+s_y+s_z) + (z & (x | y) & "s"_mst) | // s=s_z*(s_x+s_y) + "x"_mst | // x + ((x | y | z) & "ghij"_mst) | // g=g_x+g_y+g_z, h=h_x+h_y+h_z, i=i_x+i_y+i_z, j=j_x+j_y_j_z + "k"_mst.If(((x & y & z) << "k"_mst) && + !(((x << "g"_mst) && (y << "h"_mst)) || + ((x << "h"_mst) && (y << "g"_mst)) || + ((x << "i"_mst) && (y << "j"_mst)) || + ((x << "j"_mst) && (y << "i"_mst)))); // k=k_x*k_y*k_z* !(g_x*h_y + h_x*g_y + i_x*j_y + j_x*i_y) + case Fragment::MULTI: return "Bnudemsk"_mst; + case Fragment::THRESH: { + bool all_e = true; + bool all_m = true; + uint32_t args = 0; + uint32_t num_s = 0; + Type acc_tl = "k"_mst; + for (size_t i = 0; i < sub_types.size(); ++i) { + Type t = sub_types[i]; + if (!(t << (i ? "Wdu"_mst : "Bdu"_mst))) return ""_mst; // Require Bdu, Wdu, Wdu, ... + if (!(t << "e"_mst)) all_e = false; + if (!(t << "m"_mst)) all_m = false; + if (t << "s"_mst) num_s += 1; + args += (t << "z"_mst) ? 0 : (t << "o"_mst) ? 1 : 2; + acc_tl = ((acc_tl | t) & "ghij"_mst) | + // Thresh contains a combination of timelocks if it has threshold > 1 and + // it contains two different children that have different types of timelocks + // Note how if any of the children don't have "k", the parent also does not have "k" + "k"_mst.If(((acc_tl & t) << "k"_mst) && ((k <= 1) || + ((k > 1) && !(((acc_tl << "g"_mst) && (t << "h"_mst)) || + ((acc_tl << "h"_mst) && (t << "g"_mst)) || + ((acc_tl << "i"_mst) && (t << "j"_mst)) || + ((acc_tl << "j"_mst) && (t << "i"_mst)))))); + } + return "Bdu"_mst | + "z"_mst.If(args == 0) | // z=all z + "o"_mst.If(args == 1) | // o=all z except one o + "e"_mst.If(all_e && num_s == n_subs) | // e=all e and all s + "m"_mst.If(all_e && all_m && num_s >= n_subs - k) | // m=all e, >=(n-k) s + "s"_mst.If(num_s >= n_subs - k + 1) | // s= >=(n-k+1) s + acc_tl; // timelock info + } + } + assert(false); + return ""_mst; +} + +size_t ComputeScriptLen(Fragment nodetype, Type sub0typ, size_t subsize, uint32_t k, size_t n_subs, size_t n_keys) { + switch (nodetype) { + case Fragment::JUST_1: + case Fragment::JUST_0: return 1; + case Fragment::PK_K: return 34; + case Fragment::PK_H: return 3 + 21; + case Fragment::OLDER: + case Fragment::AFTER: return 1 + BuildScript(k).size(); + case Fragment::HASH256: + case Fragment::SHA256: return 4 + 2 + 33; + case Fragment::HASH160: + case Fragment::RIPEMD160: return 4 + 2 + 21; + case Fragment::MULTI: return 3 + (n_keys > 16) + (k > 16) + 34 * n_keys; + case Fragment::AND_V: return subsize; + case Fragment::WRAP_V: return subsize + (sub0typ << "x"_mst); + case Fragment::WRAP_S: + case Fragment::WRAP_C: + case Fragment::WRAP_N: + case Fragment::AND_B: + case Fragment::OR_B: return subsize + 1; + case Fragment::WRAP_A: + case Fragment::OR_C: return subsize + 2; + case Fragment::WRAP_D: + case Fragment::OR_D: + case Fragment::OR_I: + case Fragment::ANDOR: return subsize + 3; + case Fragment::WRAP_J: return subsize + 4; + case Fragment::THRESH: return subsize + n_subs + BuildScript(k).size(); + } + assert(false); + return 0; +} + +bool DecomposeScript(const CScript& script, std::vector<std::pair<opcodetype, std::vector<unsigned char>>>& out) +{ + out.clear(); + CScript::const_iterator it = script.begin(), itend = script.end(); + while (it != itend) { + std::vector<unsigned char> push_data; + opcodetype opcode; + if (!script.GetOp(it, opcode, push_data)) { + out.clear(); + return false; + } else if (opcode >= OP_1 && opcode <= OP_16) { + // Deal with OP_n (GetOp does not turn them into pushes). + push_data.assign(1, CScript::DecodeOP_N(opcode)); + } else if (opcode == OP_CHECKSIGVERIFY) { + // Decompose OP_CHECKSIGVERIFY into OP_CHECKSIG OP_VERIFY + out.emplace_back(OP_CHECKSIG, std::vector<unsigned char>()); + opcode = OP_VERIFY; + } else if (opcode == OP_CHECKMULTISIGVERIFY) { + // Decompose OP_CHECKMULTISIGVERIFY into OP_CHECKMULTISIG OP_VERIFY + out.emplace_back(OP_CHECKMULTISIG, std::vector<unsigned char>()); + opcode = OP_VERIFY; + } else if (opcode == OP_EQUALVERIFY) { + // Decompose OP_EQUALVERIFY into OP_EQUAL OP_VERIFY + out.emplace_back(OP_EQUAL, std::vector<unsigned char>()); + opcode = OP_VERIFY; + } else if (IsPushdataOp(opcode)) { + if (!CheckMinimalPush(push_data, opcode)) return false; + } else if (it != itend && (opcode == OP_CHECKSIG || opcode == OP_CHECKMULTISIG || opcode == OP_EQUAL) && (*it == OP_VERIFY)) { + // Rule out non minimal VERIFY sequences + return false; + } + out.emplace_back(opcode, std::move(push_data)); + } + std::reverse(out.begin(), out.end()); + return true; +} + +bool ParseScriptNumber(const std::pair<opcodetype, std::vector<unsigned char>>& in, int64_t& k) { + if (in.first == OP_0) { + k = 0; + return true; + } + if (!in.second.empty()) { + if (IsPushdataOp(in.first) && !CheckMinimalPush(in.second, in.first)) return false; + try { + k = CScriptNum(in.second, true).GetInt64(); + return true; + } catch(const scriptnum_error&) {} + } + return false; +} + +int FindNextChar(Span<const char> sp, const char m) +{ + for (int i = 0; i < (int)sp.size(); ++i) { + if (sp[i] == m) return i; + // We only search within the current parentheses + if (sp[i] == ')') break; + } + return -1; +} + +} // namespace internal +} // namespace miniscript diff --git a/src/script/miniscript.h b/src/script/miniscript.h new file mode 100644 index 0000000000..b54653c548 --- /dev/null +++ b/src/script/miniscript.h @@ -0,0 +1,1652 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_SCRIPT_MINISCRIPT_H +#define BITCOIN_SCRIPT_MINISCRIPT_H + +#include <algorithm> +#include <numeric> +#include <memory> +#include <optional> +#include <string> +#include <vector> + +#include <stdlib.h> +#include <assert.h> + +#include <policy/policy.h> +#include <primitives/transaction.h> +#include <script/script.h> +#include <span.h> +#include <util/spanparsing.h> +#include <util/strencodings.h> +#include <util/string.h> +#include <util/vector.h> + +namespace miniscript { + +/** This type encapsulates the miniscript type system properties. + * + * Every miniscript expression is one of 4 basic types, and additionally has + * a number of boolean type properties. + * + * The basic types are: + * - "B" Base: + * - Takes its inputs from the top of the stack. + * - When satisfied, pushes a nonzero value of up to 4 bytes onto the stack. + * - When dissatisfied, pushes a 0 onto the stack. + * - This is used for most expressions, and required for the top level one. + * - For example: older(n) = <n> OP_CHECKSEQUENCEVERIFY. + * - "V" Verify: + * - Takes its inputs from the top of the stack. + * - When satisfactied, pushes nothing. + * - Cannot be dissatisfied. + * - This can be obtained by adding an OP_VERIFY to a B, modifying the last opcode + * of a B to its -VERIFY version (only for OP_CHECKSIG, OP_CHECKSIGVERIFY + * and OP_EQUAL), or by combining a V fragment under some conditions. + * - For example vc:pk_k(key) = <key> OP_CHECKSIGVERIFY + * - "K" Key: + * - Takes its inputs from the top of the stack. + * - Becomes a B when followed by OP_CHECKSIG. + * - Always pushes a public key onto the stack, for which a signature is to be + * provided to satisfy the expression. + * - For example pk_h(key) = OP_DUP OP_HASH160 <Hash160(key)> OP_EQUALVERIFY + * - "W" Wrapped: + * - Takes its input from one below the top of the stack. + * - When satisfied, pushes a nonzero value (like B) on top of the stack, or one below. + * - When dissatisfied, pushes 0 op top of the stack or one below. + * - Is always "OP_SWAP [B]" or "OP_TOALTSTACK [B] OP_FROMALTSTACK". + * - For example sc:pk_k(key) = OP_SWAP <key> OP_CHECKSIG + * + * There a type properties that help reasoning about correctness: + * - "z" Zero-arg: + * - Is known to always consume exactly 0 stack elements. + * - For example after(n) = <n> OP_CHECKLOCKTIMEVERIFY + * - "o" One-arg: + * - Is known to always consume exactly 1 stack element. + * - Conflicts with property 'z' + * - For example sha256(hash) = OP_SIZE 32 OP_EQUALVERIFY OP_SHA256 <hash> OP_EQUAL + * - "n" Nonzero: + * - For every way this expression can be satisfied, a satisfaction exists that never needs + * a zero top stack element. + * - Conflicts with property 'z' and with type 'W'. + * - "d" Dissatisfiable: + * - There is an easy way to construct a dissatisfaction for this expression. + * - Conflicts with type 'V'. + * - "u" Unit: + * - In case of satisfaction, an exact 1 is put on the stack (rather than just nonzero). + * - Conflicts with type 'V'. + * + * Additional type properties help reasoning about nonmalleability: + * - "e" Expression: + * - This implies property 'd', but the dissatisfaction is nonmalleable. + * - This generally requires 'e' for all subexpressions which are invoked for that + * dissatifsaction, and property 'f' for the unexecuted subexpressions in that case. + * - Conflicts with type 'V'. + * - "f" Forced: + * - Dissatisfactions (if any) for this expression always involve at least one signature. + * - Is always true for type 'V'. + * - "s" Safe: + * - Satisfactions for this expression always involve at least one signature. + * - "m" Nonmalleable: + * - For every way this expression can be satisfied (which may be none), + * a nonmalleable satisfaction exists. + * - This generally requires 'm' for all subexpressions, and 'e' for all subexpressions + * which are dissatisfied when satisfying the parent. + * + * One type property is an implementation detail: + * - "x" Expensive verify: + * - Expressions with this property have a script whose last opcode is not EQUAL, CHECKSIG, or CHECKMULTISIG. + * - Not having this property means that it can be converted to a V at no cost (by switching to the + * -VERIFY version of the last opcode). + * + * Five more type properties for representing timelock information. Spend paths + * in miniscripts containing conflicting timelocks and heightlocks cannot be spent together. + * This helps users detect if miniscript does not match the semantic behaviour the + * user expects. + * - "g" Whether the branch contains a relative time timelock + * - "h" Whether the branch contains a relative height timelock + * - "i" Whether the branch contains an absolute time timelock + * - "j" Whether the branch contains an absolute height timelock + * - "k" + * - Whether all satisfactions of this expression don't contain a mix of heightlock and timelock + * of the same type. + * - If the miniscript does not have the "k" property, the miniscript template will not match + * the user expectation of the corresponding spending policy. + * For each of these properties the subset rule holds: an expression with properties X, Y, and Z, is also + * valid in places where an X, a Y, a Z, an XY, ... is expected. +*/ +class Type { + //! Internal bitmap of properties (see ""_mst operator for details). + uint32_t m_flags; + + //! Internal constructor used by the ""_mst operator. + explicit constexpr Type(uint32_t flags) : m_flags(flags) {} + +public: + //! The only way to publicly construct a Type is using this literal operator. + friend constexpr Type operator"" _mst(const char* c, size_t l); + + //! Compute the type with the union of properties. + constexpr Type operator|(Type x) const { return Type(m_flags | x.m_flags); } + + //! Compute the type with the intersection of properties. + constexpr Type operator&(Type x) const { return Type(m_flags & x.m_flags); } + + //! Check whether the left hand's properties are superset of the right's (= left is a subtype of right). + constexpr bool operator<<(Type x) const { return (x.m_flags & ~m_flags) == 0; } + + //! Comparison operator to enable use in sets/maps (total ordering incompatible with <<). + constexpr bool operator<(Type x) const { return m_flags < x.m_flags; } + + //! Equality operator. + constexpr bool operator==(Type x) const { return m_flags == x.m_flags; } + + //! The empty type if x is false, itself otherwise. + constexpr Type If(bool x) const { return Type(x ? m_flags : 0); } +}; + +//! Literal operator to construct Type objects. +inline constexpr Type operator"" _mst(const char* c, size_t l) { + Type typ{0}; + + for (const char *p = c; p < c + l; p++) { + typ = typ | Type( + *p == 'B' ? 1 << 0 : // Base type + *p == 'V' ? 1 << 1 : // Verify type + *p == 'K' ? 1 << 2 : // Key type + *p == 'W' ? 1 << 3 : // Wrapped type + *p == 'z' ? 1 << 4 : // Zero-arg property + *p == 'o' ? 1 << 5 : // One-arg property + *p == 'n' ? 1 << 6 : // Nonzero arg property + *p == 'd' ? 1 << 7 : // Dissatisfiable property + *p == 'u' ? 1 << 8 : // Unit property + *p == 'e' ? 1 << 9 : // Expression property + *p == 'f' ? 1 << 10 : // Forced property + *p == 's' ? 1 << 11 : // Safe property + *p == 'm' ? 1 << 12 : // Nonmalleable property + *p == 'x' ? 1 << 13 : // Expensive verify + *p == 'g' ? 1 << 14 : // older: contains relative time timelock (csv_time) + *p == 'h' ? 1 << 15 : // older: contains relative height timelock (csv_height) + *p == 'i' ? 1 << 16 : // after: contains time timelock (cltv_time) + *p == 'j' ? 1 << 17 : // after: contains height timelock (cltv_height) + *p == 'k' ? 1 << 18 : // does not contain a combination of height and time locks + (throw std::logic_error("Unknown character in _mst literal"), 0) + ); + } + + return typ; +} + +template<typename Key> struct Node; +template<typename Key> using NodeRef = std::shared_ptr<const Node<Key>>; + +//! Construct a miniscript node as a shared_ptr. +template<typename Key, typename... Args> +NodeRef<Key> MakeNodeRef(Args&&... args) { return std::make_shared<const Node<Key>>(std::forward<Args>(args)...); } + +//! The different node types in miniscript. +enum class Fragment { + JUST_0, //!< OP_0 + JUST_1, //!< OP_1 + PK_K, //!< [key] + PK_H, //!< OP_DUP OP_HASH160 [keyhash] OP_EQUALVERIFY + OLDER, //!< [n] OP_CHECKSEQUENCEVERIFY + AFTER, //!< [n] OP_CHECKLOCKTIMEVERIFY + SHA256, //!< OP_SIZE 32 OP_EQUALVERIFY OP_SHA256 [hash] OP_EQUAL + HASH256, //!< OP_SIZE 32 OP_EQUALVERIFY OP_HASH256 [hash] OP_EQUAL + RIPEMD160, //!< OP_SIZE 32 OP_EQUALVERIFY OP_RIPEMD160 [hash] OP_EQUAL + HASH160, //!< OP_SIZE 32 OP_EQUALVERIFY OP_HASH160 [hash] OP_EQUAL + WRAP_A, //!< OP_TOALTSTACK [X] OP_FROMALTSTACK + WRAP_S, //!< OP_SWAP [X] + WRAP_C, //!< [X] OP_CHECKSIG + WRAP_D, //!< OP_DUP OP_IF [X] OP_ENDIF + WRAP_V, //!< [X] OP_VERIFY (or -VERIFY version of last opcode in X) + WRAP_J, //!< OP_SIZE OP_0NOTEQUAL OP_IF [X] OP_ENDIF + WRAP_N, //!< [X] OP_0NOTEQUAL + AND_V, //!< [X] [Y] + AND_B, //!< [X] [Y] OP_BOOLAND + OR_B, //!< [X] [Y] OP_BOOLOR + OR_C, //!< [X] OP_NOTIF [Y] OP_ENDIF + OR_D, //!< [X] OP_IFDUP OP_NOTIF [Y] OP_ENDIF + OR_I, //!< OP_IF [X] OP_ELSE [Y] OP_ENDIF + ANDOR, //!< [X] OP_NOTIF [Z] OP_ELSE [Y] OP_ENDIF + THRESH, //!< [X1] ([Xn] OP_ADD)* [k] OP_EQUAL + MULTI, //!< [k] [key_n]* [n] OP_CHECKMULTISIG + // AND_N(X,Y) is represented as ANDOR(X,Y,0) + // WRAP_T(X) is represented as AND_V(X,1) + // WRAP_L(X) is represented as OR_I(0,X) + // WRAP_U(X) is represented as OR_I(X,0) +}; + + +namespace internal { + +//! Helper function for Node::CalcType. +Type ComputeType(Fragment nodetype, Type x, Type y, Type z, const std::vector<Type>& sub_types, uint32_t k, size_t data_size, size_t n_subs, size_t n_keys); + +//! Helper function for Node::CalcScriptLen. +size_t ComputeScriptLen(Fragment nodetype, Type sub0typ, size_t subsize, uint32_t k, size_t n_subs, size_t n_keys); + +//! A helper sanitizer/checker for the output of CalcType. +Type SanitizeType(Type x); + +//! Class whose objects represent the maximum of a list of integers. +template<typename I> +struct MaxInt { + const bool valid; + const I value; + + MaxInt() : valid(false), value(0) {} + MaxInt(I val) : valid(true), value(val) {} + + friend MaxInt<I> operator+(const MaxInt<I>& a, const MaxInt<I>& b) { + if (!a.valid || !b.valid) return {}; + return a.value + b.value; + } + + friend MaxInt<I> operator|(const MaxInt<I>& a, const MaxInt<I>& b) { + if (!a.valid) return b; + if (!b.valid) return a; + return std::max(a.value, b.value); + } +}; + +struct Ops { + //! Non-push opcodes. + uint32_t count; + //! Number of keys in possibly executed OP_CHECKMULTISIG(VERIFY)s to satisfy. + MaxInt<uint32_t> sat; + //! Number of keys in possibly executed OP_CHECKMULTISIG(VERIFY)s to dissatisfy. + MaxInt<uint32_t> dsat; + + Ops(uint32_t in_count, MaxInt<uint32_t> in_sat, MaxInt<uint32_t> in_dsat) : count(in_count), sat(in_sat), dsat(in_dsat) {}; +}; + +struct StackSize { + //! Maximum stack size to satisfy; + MaxInt<uint32_t> sat; + //! Maximum stack size to dissatisfy; + MaxInt<uint32_t> dsat; + + StackSize(MaxInt<uint32_t> in_sat, MaxInt<uint32_t> in_dsat) : sat(in_sat), dsat(in_dsat) {}; +}; + +} // namespace internal + +//! A node in a miniscript expression. +template<typename Key> +struct Node { + //! What node type this node is. + const Fragment nodetype; + //! The k parameter (time for OLDER/AFTER, threshold for THRESH(_M)) + const uint32_t k = 0; + //! The keys used by this expression (only for PK_K/PK_H/MULTI) + const std::vector<Key> keys; + //! The data bytes in this expression (only for HASH160/HASH256/SHA256/RIPEMD10). + const std::vector<unsigned char> data; + //! Subexpressions (for WRAP_*/AND_*/OR_*/ANDOR/THRESH) + const std::vector<NodeRef<Key>> subs; + +private: + //! Cached ops counts. + const internal::Ops ops; + //! Cached stack size bounds. + const internal::StackSize ss; + //! Cached expression type (computed by CalcType and fed through SanitizeType). + const Type typ; + //! Cached script length (computed by CalcScriptLen). + const size_t scriptlen; + + //! Compute the length of the script for this miniscript (including children). + size_t CalcScriptLen() const { + size_t subsize = 0; + for (const auto& sub : subs) { + subsize += sub->ScriptSize(); + } + Type sub0type = subs.size() > 0 ? subs[0]->GetType() : ""_mst; + return internal::ComputeScriptLen(nodetype, sub0type, subsize, k, subs.size(), keys.size()); + } + + /* Apply a recursive algorithm to a Miniscript tree, without actual recursive calls. + * + * The algorithm is defined by two functions: downfn and upfn. Conceptually, the + * result can be thought of as first using downfn to compute a "state" for each node, + * from the root down to the leaves. Then upfn is used to compute a "result" for each + * node, from the leaves back up to the root, which is then returned. In the actual + * implementation, both functions are invoked in an interleaved fashion, performing a + * depth-first traversal of the tree. + * + * In more detail, it is invoked as node.TreeEvalMaybe<Result>(root, downfn, upfn): + * - root is the state of the root node, of type State. + * - downfn is a callable (State&, const Node&, size_t) -> State, which given a + * node, its state, and an index of one of its children, computes the state of that + * child. It can modify the state. Children of a given node will have downfn() + * called in order. + * - upfn is a callable (State&&, const Node&, Span<Result>) -> std::optional<Result>, + * which given a node, its state, and a Span of the results of its children, + * computes the result of the node. If std::nullopt is returned by upfn, + * TreeEvalMaybe() immediately returns std::nullopt. + * The return value of TreeEvalMaybe is the result of the root node. + */ + template<typename Result, typename State, typename DownFn, typename UpFn> + std::optional<Result> TreeEvalMaybe(State root_state, DownFn downfn, UpFn upfn) const + { + /** Entries of the explicit stack tracked in this algorithm. */ + struct StackElem + { + const Node& node; //!< The node being evaluated. + size_t expanded; //!< How many children of this node have been expanded. + State state; //!< The state for that node. + + StackElem(const Node& node_, size_t exp_, State&& state_) : + node(node_), expanded(exp_), state(std::move(state_)) {} + }; + /* Stack of tree nodes being explored. */ + std::vector<StackElem> stack; + /* Results of subtrees so far. Their order and mapping to tree nodes + * is implicitly defined by stack. */ + std::vector<Result> results; + stack.emplace_back(*this, 0, std::move(root_state)); + + /* Here is a demonstration of the algorithm, for an example tree A(B,C(D,E),F). + * State variables are omitted for simplicity. + * + * First: stack=[(A,0)] results=[] + * stack=[(A,1),(B,0)] results=[] + * stack=[(A,1)] results=[B] + * stack=[(A,2),(C,0)] results=[B] + * stack=[(A,2),(C,1),(D,0)] results=[B] + * stack=[(A,2),(C,1)] results=[B,D] + * stack=[(A,2),(C,2),(E,0)] results=[B,D] + * stack=[(A,2),(C,2)] results=[B,D,E] + * stack=[(A,2)] results=[B,C] + * stack=[(A,3),(F,0)] results=[B,C] + * stack=[(A,3)] results=[B,C,F] + * Final: stack=[] results=[A] + */ + while (stack.size()) { + const Node& node = stack.back().node; + if (stack.back().expanded < node.subs.size()) { + /* We encounter a tree node with at least one unexpanded child. + * Expand it. By the time we hit this node again, the result of + * that child (and all earlier children) will be at the end of `results`. */ + size_t child_index = stack.back().expanded++; + State child_state = downfn(stack.back().state, node, child_index); + stack.emplace_back(*node.subs[child_index], 0, std::move(child_state)); + continue; + } + // Invoke upfn with the last node.subs.size() elements of results as input. + assert(results.size() >= node.subs.size()); + std::optional<Result> result{upfn(std::move(stack.back().state), node, + Span<Result>{results}.last(node.subs.size()))}; + // If evaluation returns std::nullopt, abort immediately. + if (!result) return {}; + // Replace the last node.subs.size() elements of results with the new result. + results.erase(results.end() - node.subs.size(), results.end()); + results.push_back(std::move(*result)); + stack.pop_back(); + } + // The final remaining results element is the root result, return it. + assert(results.size() == 1); + return std::move(results[0]); + } + + /** Like TreeEvalMaybe, but always produces a result. upfn must return Result. */ + template<typename Result, typename State, typename DownFn, typename UpFn> + Result TreeEval(State root_state, DownFn&& downfn, UpFn upfn) const + { + // Invoke TreeEvalMaybe with upfn wrapped to return std::optional<Result>, and then + // unconditionally dereference the result (it cannot be std::nullopt). + return std::move(*TreeEvalMaybe<Result>(std::move(root_state), + std::forward<DownFn>(downfn), + [&upfn](State&& state, const Node& node, Span<Result> subs) { + Result res{upfn(std::move(state), node, subs)}; + return std::optional<Result>(std::move(res)); + } + )); + } + + //! Compute the type for this miniscript. + Type CalcType() const { + using namespace internal; + + // THRESH has a variable number of subexpressions + std::vector<Type> sub_types; + if (nodetype == Fragment::THRESH) { + for (const auto& sub : subs) sub_types.push_back(sub->GetType()); + } + // All other nodes than THRESH can be computed just from the types of the 0-3 subexpressions. + Type x = subs.size() > 0 ? subs[0]->GetType() : ""_mst; + Type y = subs.size() > 1 ? subs[1]->GetType() : ""_mst; + Type z = subs.size() > 2 ? subs[2]->GetType() : ""_mst; + + return SanitizeType(ComputeType(nodetype, x, y, z, sub_types, k, data.size(), subs.size(), keys.size())); + } + +public: + template<typename Ctx> + CScript ToScript(const Ctx& ctx) const + { + // To construct the CScript for a Miniscript object, we use the TreeEval algorithm. + // The State is a boolean: whether or not the node's script expansion is followed + // by an OP_VERIFY (which may need to be combined with the last script opcode). + auto downfn = [](bool verify, const Node& node, size_t index) { + // For WRAP_V, the subexpression is certainly followed by OP_VERIFY. + if (node.nodetype == Fragment::WRAP_V) return true; + // The subexpression of WRAP_S, and the last subexpression of AND_V + // inherit the followed-by-OP_VERIFY property from the parent. + if (node.nodetype == Fragment::WRAP_S || + (node.nodetype == Fragment::AND_V && index == 1)) return verify; + return false; + }; + // The upward function computes for a node, given its followed-by-OP_VERIFY status + // and the CScripts of its child nodes, the CScript of the node. + auto upfn = [&ctx](bool verify, const Node& node, Span<CScript> subs) -> CScript { + switch (node.nodetype) { + case Fragment::PK_K: return BuildScript(ctx.ToPKBytes(node.keys[0])); + case Fragment::PK_H: return BuildScript(OP_DUP, OP_HASH160, ctx.ToPKHBytes(node.keys[0]), OP_EQUALVERIFY); + case Fragment::OLDER: return BuildScript(node.k, OP_CHECKSEQUENCEVERIFY); + case Fragment::AFTER: return BuildScript(node.k, OP_CHECKLOCKTIMEVERIFY); + case Fragment::SHA256: return BuildScript(OP_SIZE, 32, OP_EQUALVERIFY, OP_SHA256, node.data, verify ? OP_EQUALVERIFY : OP_EQUAL); + case Fragment::RIPEMD160: return BuildScript(OP_SIZE, 32, OP_EQUALVERIFY, OP_RIPEMD160, node.data, verify ? OP_EQUALVERIFY : OP_EQUAL); + case Fragment::HASH256: return BuildScript(OP_SIZE, 32, OP_EQUALVERIFY, OP_HASH256, node.data, verify ? OP_EQUALVERIFY : OP_EQUAL); + case Fragment::HASH160: return BuildScript(OP_SIZE, 32, OP_EQUALVERIFY, OP_HASH160, node.data, verify ? OP_EQUALVERIFY : OP_EQUAL); + case Fragment::WRAP_A: return BuildScript(OP_TOALTSTACK, subs[0], OP_FROMALTSTACK); + case Fragment::WRAP_S: return BuildScript(OP_SWAP, subs[0]); + case Fragment::WRAP_C: return BuildScript(std::move(subs[0]), verify ? OP_CHECKSIGVERIFY : OP_CHECKSIG); + case Fragment::WRAP_D: return BuildScript(OP_DUP, OP_IF, subs[0], OP_ENDIF); + case Fragment::WRAP_V: { + if (node.subs[0]->GetType() << "x"_mst) { + return BuildScript(std::move(subs[0]), OP_VERIFY); + } else { + return std::move(subs[0]); + } + } + case Fragment::WRAP_J: return BuildScript(OP_SIZE, OP_0NOTEQUAL, OP_IF, subs[0], OP_ENDIF); + case Fragment::WRAP_N: return BuildScript(std::move(subs[0]), OP_0NOTEQUAL); + case Fragment::JUST_1: return BuildScript(OP_1); + case Fragment::JUST_0: return BuildScript(OP_0); + case Fragment::AND_V: return BuildScript(std::move(subs[0]), subs[1]); + case Fragment::AND_B: return BuildScript(std::move(subs[0]), subs[1], OP_BOOLAND); + case Fragment::OR_B: return BuildScript(std::move(subs[0]), subs[1], OP_BOOLOR); + case Fragment::OR_D: return BuildScript(std::move(subs[0]), OP_IFDUP, OP_NOTIF, subs[1], OP_ENDIF); + case Fragment::OR_C: return BuildScript(std::move(subs[0]), OP_NOTIF, subs[1], OP_ENDIF); + case Fragment::OR_I: return BuildScript(OP_IF, subs[0], OP_ELSE, subs[1], OP_ENDIF); + case Fragment::ANDOR: return BuildScript(std::move(subs[0]), OP_NOTIF, subs[2], OP_ELSE, subs[1], OP_ENDIF); + case Fragment::MULTI: { + CScript script = BuildScript(node.k); + for (const auto& key : node.keys) { + script = BuildScript(std::move(script), ctx.ToPKBytes(key)); + } + return BuildScript(std::move(script), node.keys.size(), verify ? OP_CHECKMULTISIGVERIFY : OP_CHECKMULTISIG); + } + case Fragment::THRESH: { + CScript script = std::move(subs[0]); + for (size_t i = 1; i < subs.size(); ++i) { + script = BuildScript(std::move(script), subs[i], OP_ADD); + } + return BuildScript(std::move(script), node.k, verify ? OP_EQUALVERIFY : OP_EQUAL); + } + } + assert(false); + return {}; + }; + return TreeEval<CScript>(false, downfn, upfn); + } + + template<typename CTx> + bool ToString(const CTx& ctx, std::string& ret) const { + // To construct the std::string representation for a Miniscript object, we use + // the TreeEvalMaybe algorithm. The State is a boolean: whether the parent node is a + // wrapper. If so, non-wrapper expressions must be prefixed with a ":". + auto downfn = [](bool, const Node& node, size_t) { + return (node.nodetype == Fragment::WRAP_A || node.nodetype == Fragment::WRAP_S || + node.nodetype == Fragment::WRAP_D || node.nodetype == Fragment::WRAP_V || + node.nodetype == Fragment::WRAP_J || node.nodetype == Fragment::WRAP_N || + node.nodetype == Fragment::WRAP_C || + (node.nodetype == Fragment::AND_V && node.subs[1]->nodetype == Fragment::JUST_1) || + (node.nodetype == Fragment::OR_I && node.subs[0]->nodetype == Fragment::JUST_0) || + (node.nodetype == Fragment::OR_I && node.subs[1]->nodetype == Fragment::JUST_0)); + }; + // The upward function computes for a node, given whether its parent is a wrapper, + // and the string representations of its child nodes, the string representation of the node. + auto upfn = [&ctx](bool wrapped, const Node& node, Span<std::string> subs) -> std::optional<std::string> { + std::string ret = wrapped ? ":" : ""; + + switch (node.nodetype) { + case Fragment::WRAP_A: return "a" + std::move(subs[0]); + case Fragment::WRAP_S: return "s" + std::move(subs[0]); + case Fragment::WRAP_C: + if (node.subs[0]->nodetype == Fragment::PK_K) { + // pk(K) is syntactic sugar for c:pk_k(K) + std::string key_str; + if (!ctx.ToString(node.subs[0]->keys[0], key_str)) return {}; + return std::move(ret) + "pk(" + std::move(key_str) + ")"; + } + if (node.subs[0]->nodetype == Fragment::PK_H) { + // pkh(K) is syntactic sugar for c:pk_h(K) + std::string key_str; + if (!ctx.ToString(node.subs[0]->keys[0], key_str)) return {}; + return std::move(ret) + "pkh(" + std::move(key_str) + ")"; + } + return "c" + std::move(subs[0]); + case Fragment::WRAP_D: return "d" + std::move(subs[0]); + case Fragment::WRAP_V: return "v" + std::move(subs[0]); + case Fragment::WRAP_J: return "j" + std::move(subs[0]); + case Fragment::WRAP_N: return "n" + std::move(subs[0]); + case Fragment::AND_V: + // t:X is syntactic sugar for and_v(X,1). + if (node.subs[1]->nodetype == Fragment::JUST_1) return "t" + std::move(subs[0]); + break; + case Fragment::OR_I: + if (node.subs[0]->nodetype == Fragment::JUST_0) return "l" + std::move(subs[1]); + if (node.subs[1]->nodetype == Fragment::JUST_0) return "u" + std::move(subs[0]); + break; + default: break; + } + switch (node.nodetype) { + case Fragment::PK_K: { + std::string key_str; + if (!ctx.ToString(node.keys[0], key_str)) return {}; + return std::move(ret) + "pk_k(" + std::move(key_str) + ")"; + } + case Fragment::PK_H: { + std::string key_str; + if (!ctx.ToString(node.keys[0], key_str)) return {}; + return std::move(ret) + "pk_h(" + std::move(key_str) + ")"; + } + case Fragment::AFTER: return std::move(ret) + "after(" + ::ToString(node.k) + ")"; + case Fragment::OLDER: return std::move(ret) + "older(" + ::ToString(node.k) + ")"; + case Fragment::HASH256: return std::move(ret) + "hash256(" + HexStr(node.data) + ")"; + case Fragment::HASH160: return std::move(ret) + "hash160(" + HexStr(node.data) + ")"; + case Fragment::SHA256: return std::move(ret) + "sha256(" + HexStr(node.data) + ")"; + case Fragment::RIPEMD160: return std::move(ret) + "ripemd160(" + HexStr(node.data) + ")"; + case Fragment::JUST_1: return std::move(ret) + "1"; + case Fragment::JUST_0: return std::move(ret) + "0"; + case Fragment::AND_V: return std::move(ret) + "and_v(" + std::move(subs[0]) + "," + std::move(subs[1]) + ")"; + case Fragment::AND_B: return std::move(ret) + "and_b(" + std::move(subs[0]) + "," + std::move(subs[1]) + ")"; + case Fragment::OR_B: return std::move(ret) + "or_b(" + std::move(subs[0]) + "," + std::move(subs[1]) + ")"; + case Fragment::OR_D: return std::move(ret) + "or_d(" + std::move(subs[0]) + "," + std::move(subs[1]) + ")"; + case Fragment::OR_C: return std::move(ret) + "or_c(" + std::move(subs[0]) + "," + std::move(subs[1]) + ")"; + case Fragment::OR_I: return std::move(ret) + "or_i(" + std::move(subs[0]) + "," + std::move(subs[1]) + ")"; + case Fragment::ANDOR: + // and_n(X,Y) is syntactic sugar for andor(X,Y,0). + if (node.subs[2]->nodetype == Fragment::JUST_0) return std::move(ret) + "and_n(" + std::move(subs[0]) + "," + std::move(subs[1]) + ")"; + return std::move(ret) + "andor(" + std::move(subs[0]) + "," + std::move(subs[1]) + "," + std::move(subs[2]) + ")"; + case Fragment::MULTI: { + auto str = std::move(ret) + "multi(" + ::ToString(node.k); + for (const auto& key : node.keys) { + std::string key_str; + if (!ctx.ToString(key, key_str)) return {}; + str += "," + std::move(key_str); + } + return std::move(str) + ")"; + } + case Fragment::THRESH: { + auto str = std::move(ret) + "thresh(" + ::ToString(node.k); + for (auto& sub : subs) { + str += "," + std::move(sub); + } + return std::move(str) + ")"; + } + default: assert(false); + } + return ""; // Should never be reached. + }; + + auto res = TreeEvalMaybe<std::string>(false, downfn, upfn); + if (res.has_value()) ret = std::move(*res); + return res.has_value(); + } + + internal::Ops CalcOps() const { + switch (nodetype) { + case Fragment::JUST_1: return {0, 0, {}}; + case Fragment::JUST_0: return {0, {}, 0}; + case Fragment::PK_K: return {0, 0, 0}; + case Fragment::PK_H: return {3, 0, 0}; + case Fragment::OLDER: + case Fragment::AFTER: return {1, 0, {}}; + case Fragment::SHA256: + case Fragment::RIPEMD160: + case Fragment::HASH256: + case Fragment::HASH160: return {4, 0, {}}; + case Fragment::AND_V: return {subs[0]->ops.count + subs[1]->ops.count, subs[0]->ops.sat + subs[1]->ops.sat, {}}; + case Fragment::AND_B: { + const auto count{1 + subs[0]->ops.count + subs[1]->ops.count}; + const auto sat{subs[0]->ops.sat + subs[1]->ops.sat}; + const auto dsat{subs[0]->ops.dsat + subs[1]->ops.dsat}; + return {count, sat, dsat}; + } + case Fragment::OR_B: { + const auto count{1 + subs[0]->ops.count + subs[1]->ops.count}; + const auto sat{(subs[0]->ops.sat + subs[1]->ops.dsat) | (subs[1]->ops.sat + subs[0]->ops.dsat)}; + const auto dsat{subs[0]->ops.dsat + subs[1]->ops.dsat}; + return {count, sat, dsat}; + } + case Fragment::OR_D: { + const auto count{3 + subs[0]->ops.count + subs[1]->ops.count}; + const auto sat{subs[0]->ops.sat | (subs[1]->ops.sat + subs[0]->ops.dsat)}; + const auto dsat{subs[0]->ops.dsat + subs[1]->ops.dsat}; + return {count, sat, dsat}; + } + case Fragment::OR_C: { + const auto count{2 + subs[0]->ops.count + subs[1]->ops.count}; + const auto sat{subs[0]->ops.sat | (subs[1]->ops.sat + subs[0]->ops.dsat)}; + return {count, sat, {}}; + } + case Fragment::OR_I: { + const auto count{3 + subs[0]->ops.count + subs[1]->ops.count}; + const auto sat{subs[0]->ops.sat | subs[1]->ops.sat}; + const auto dsat{subs[0]->ops.dsat | subs[1]->ops.dsat}; + return {count, sat, dsat}; + } + case Fragment::ANDOR: { + const auto count{3 + subs[0]->ops.count + subs[1]->ops.count + subs[2]->ops.count}; + const auto sat{(subs[1]->ops.sat + subs[0]->ops.sat) | (subs[0]->ops.dsat + subs[2]->ops.sat)}; + const auto dsat{subs[0]->ops.dsat + subs[2]->ops.dsat}; + return {count, sat, dsat}; + } + case Fragment::MULTI: return {1, (uint32_t)keys.size(), (uint32_t)keys.size()}; + case Fragment::WRAP_S: + case Fragment::WRAP_C: + case Fragment::WRAP_N: return {1 + subs[0]->ops.count, subs[0]->ops.sat, subs[0]->ops.dsat}; + case Fragment::WRAP_A: return {2 + subs[0]->ops.count, subs[0]->ops.sat, subs[0]->ops.dsat}; + case Fragment::WRAP_D: return {3 + subs[0]->ops.count, subs[0]->ops.sat, 0}; + case Fragment::WRAP_J: return {4 + subs[0]->ops.count, subs[0]->ops.sat, 0}; + case Fragment::WRAP_V: return {subs[0]->ops.count + (subs[0]->GetType() << "x"_mst), subs[0]->ops.sat, {}}; + case Fragment::THRESH: { + uint32_t count = 0; + auto sats = Vector(internal::MaxInt<uint32_t>(0)); + for (const auto& sub : subs) { + count += sub->ops.count + 1; + auto next_sats = Vector(sats[0] + sub->ops.dsat); + for (size_t j = 1; j < sats.size(); ++j) next_sats.push_back((sats[j] + sub->ops.dsat) | (sats[j - 1] + sub->ops.sat)); + next_sats.push_back(sats[sats.size() - 1] + sub->ops.sat); + sats = std::move(next_sats); + } + assert(k <= sats.size()); + return {count, sats[k], sats[0]}; + } + } + assert(false); + return {0, {}, {}}; + } + + internal::StackSize CalcStackSize() const { + switch (nodetype) { + case Fragment::JUST_0: return {{}, 0}; + case Fragment::JUST_1: + case Fragment::OLDER: + case Fragment::AFTER: return {0, {}}; + case Fragment::PK_K: return {1, 1}; + case Fragment::PK_H: return {2, 2}; + case Fragment::SHA256: + case Fragment::RIPEMD160: + case Fragment::HASH256: + case Fragment::HASH160: return {1, {}}; + case Fragment::ANDOR: { + const auto sat{(subs[0]->ss.sat + subs[1]->ss.sat) | (subs[0]->ss.dsat + subs[2]->ss.sat)}; + const auto dsat{subs[0]->ss.dsat + subs[2]->ss.dsat}; + return {sat, dsat}; + } + case Fragment::AND_V: return {subs[0]->ss.sat + subs[1]->ss.sat, {}}; + case Fragment::AND_B: return {subs[0]->ss.sat + subs[1]->ss.sat, subs[0]->ss.dsat + subs[1]->ss.dsat}; + case Fragment::OR_B: { + const auto sat{(subs[0]->ss.dsat + subs[1]->ss.sat) | (subs[0]->ss.sat + subs[1]->ss.dsat)}; + const auto dsat{subs[0]->ss.dsat + subs[1]->ss.dsat}; + return {sat, dsat}; + } + case Fragment::OR_C: return {subs[0]->ss.sat | (subs[0]->ss.dsat + subs[1]->ss.sat), {}}; + case Fragment::OR_D: return {subs[0]->ss.sat | (subs[0]->ss.dsat + subs[1]->ss.sat), subs[0]->ss.dsat + subs[1]->ss.dsat}; + case Fragment::OR_I: return {(subs[0]->ss.sat + 1) | (subs[1]->ss.sat + 1), (subs[0]->ss.dsat + 1) | (subs[1]->ss.dsat + 1)}; + case Fragment::MULTI: return {k + 1, k + 1}; + case Fragment::WRAP_A: + case Fragment::WRAP_N: + case Fragment::WRAP_S: + case Fragment::WRAP_C: return subs[0]->ss; + case Fragment::WRAP_D: return {1 + subs[0]->ss.sat, 1}; + case Fragment::WRAP_V: return {subs[0]->ss.sat, {}}; + case Fragment::WRAP_J: return {subs[0]->ss.sat, 1}; + case Fragment::THRESH: { + auto sats = Vector(internal::MaxInt<uint32_t>(0)); + for (const auto& sub : subs) { + auto next_sats = Vector(sats[0] + sub->ss.dsat); + for (size_t j = 1; j < sats.size(); ++j) next_sats.push_back((sats[j] + sub->ss.dsat) | (sats[j - 1] + sub->ss.sat)); + next_sats.push_back(sats[sats.size() - 1] + sub->ss.sat); + sats = std::move(next_sats); + } + assert(k <= sats.size()); + return {sats[k], sats[0]}; + } + } + assert(false); + return {{}, {}}; + } + +public: + //! Return the size of the script for this expression (faster than ToScript().size()). + size_t ScriptSize() const { return scriptlen; } + + //! Return the maximum number of ops needed to satisfy this script non-malleably. + uint32_t GetOps() const { return ops.count + ops.sat.value; } + + //! Check the ops limit of this script against the consensus limit. + bool CheckOpsLimit() const { return GetOps() <= MAX_OPS_PER_SCRIPT; } + + /** Return the maximum number of stack elements needed to satisfy this script non-malleably, including + * the script push. */ + uint32_t GetStackSize() const { return ss.sat.value + 1; } + + //! Check the maximum stack size for this script against the policy limit. + bool CheckStackSize() const { return GetStackSize() - 1 <= MAX_STANDARD_P2WSH_STACK_ITEMS; } + + //! Return the expression type. + Type GetType() const { return typ; } + + //! Check whether this node is valid at all. + bool IsValid() const { return !(GetType() == ""_mst) && ScriptSize() <= MAX_STANDARD_P2WSH_SCRIPT_SIZE; } + + //! Check whether this node is valid as a script on its own. + bool IsValidTopLevel() const { return IsValid() && GetType() << "B"_mst; } + + //! Check whether this script can always be satisfied in a non-malleable way. + bool IsNonMalleable() const { return GetType() << "m"_mst; } + + //! Check whether this script always needs a signature. + bool NeedsSignature() const { return GetType() << "s"_mst; } + + //! Do all sanity checks. + bool IsSane() const { return IsValid() && GetType() << "mk"_mst && CheckOpsLimit() && CheckStackSize(); } + + //! Check whether this node is safe as a script on its own. + bool IsSaneTopLevel() const { return IsValidTopLevel() && IsSane() && NeedsSignature(); } + + //! Equality testing. + bool operator==(const Node<Key>& arg) const + { + if (nodetype != arg.nodetype) return false; + if (k != arg.k) return false; + if (data != arg.data) return false; + if (keys != arg.keys) return false; + if (subs.size() != arg.subs.size()) return false; + for (size_t i = 0; i < subs.size(); ++i) { + if (!(*subs[i] == *arg.subs[i])) return false; + } + assert(scriptlen == arg.scriptlen); + assert(typ == arg.typ); + return true; + } + + // Constructors with various argument combinations. + Node(Fragment nt, std::vector<NodeRef<Key>> sub, std::vector<unsigned char> arg, uint32_t val = 0) : nodetype(nt), k(val), data(std::move(arg)), subs(std::move(sub)), ops(CalcOps()), ss(CalcStackSize()), typ(CalcType()), scriptlen(CalcScriptLen()) {} + Node(Fragment nt, std::vector<unsigned char> arg, uint32_t val = 0) : nodetype(nt), k(val), data(std::move(arg)), ops(CalcOps()), ss(CalcStackSize()), typ(CalcType()), scriptlen(CalcScriptLen()) {} + Node(Fragment nt, std::vector<NodeRef<Key>> sub, std::vector<Key> key, uint32_t val = 0) : nodetype(nt), k(val), keys(std::move(key)), subs(std::move(sub)), ops(CalcOps()), ss(CalcStackSize()), typ(CalcType()), scriptlen(CalcScriptLen()) {} + Node(Fragment nt, std::vector<Key> key, uint32_t val = 0) : nodetype(nt), k(val), keys(std::move(key)), ops(CalcOps()), ss(CalcStackSize()), typ(CalcType()), scriptlen(CalcScriptLen()) {} + Node(Fragment nt, std::vector<NodeRef<Key>> sub, uint32_t val = 0) : nodetype(nt), k(val), subs(std::move(sub)), ops(CalcOps()), ss(CalcStackSize()), typ(CalcType()), scriptlen(CalcScriptLen()) {} + Node(Fragment nt, uint32_t val = 0) : nodetype(nt), k(val), ops(CalcOps()), ss(CalcStackSize()), typ(CalcType()), scriptlen(CalcScriptLen()) {} +}; + +namespace internal { + +enum class ParseContext { + /** An expression which may be begin with wrappers followed by a colon. */ + WRAPPED_EXPR, + /** A miniscript expression which does not begin with wrappers. */ + EXPR, + + /** SWAP wraps the top constructed node with s: */ + SWAP, + /** ALT wraps the top constructed node with a: */ + ALT, + /** CHECK wraps the top constructed node with c: */ + CHECK, + /** DUP_IF wraps the top constructed node with d: */ + DUP_IF, + /** VERIFY wraps the top constructed node with v: */ + VERIFY, + /** NON_ZERO wraps the top constructed node with j: */ + NON_ZERO, + /** ZERO_NOTEQUAL wraps the top constructed node with n: */ + ZERO_NOTEQUAL, + /** WRAP_U will construct an or_i(X,0) node from the top constructed node. */ + WRAP_U, + /** WRAP_T will construct an and_v(X,1) node from the top constructed node. */ + WRAP_T, + + /** AND_N will construct an andor(X,Y,0) node from the last two constructed nodes. */ + AND_N, + /** AND_V will construct an and_v node from the last two constructed nodes. */ + AND_V, + /** AND_B will construct an and_b node from the last two constructed nodes. */ + AND_B, + /** ANDOR will construct an andor node from the last three constructed nodes. */ + ANDOR, + /** OR_B will construct an or_b node from the last two constructed nodes. */ + OR_B, + /** OR_C will construct an or_c node from the last two constructed nodes. */ + OR_C, + /** OR_D will construct an or_d node from the last two constructed nodes. */ + OR_D, + /** OR_I will construct an or_i node from the last two constructed nodes. */ + OR_I, + + /** THRESH will read a wrapped expression, and then look for a COMMA. If + * no comma follows, it will construct a thresh node from the appropriate + * number of constructed children. Otherwise, it will recurse with another + * THRESH. */ + THRESH, + + /** COMMA expects the next element to be ',' and fails if not. */ + COMMA, + /** CLOSE_BRACKET expects the next element to be ')' and fails if not. */ + CLOSE_BRACKET, +}; + +int FindNextChar(Span<const char> in, const char m); + +/** Parse a key string ending with a ')' or ','. */ +template<typename Key, typename Ctx> +std::optional<std::pair<Key, int>> ParseKeyEnd(Span<const char> in, const Ctx& ctx) +{ + Key key; + int key_size = FindNextChar(in, ')'); + if (key_size < 1) return {}; + if (!ctx.FromString(in.begin(), in.begin() + key_size, key)) return {}; + return {{std::move(key), key_size}}; +} + +/** Parse a hex string ending at the end of the fragment's text representation. */ +template<typename Ctx> +std::optional<std::pair<std::vector<unsigned char>, int>> ParseHexStrEnd(Span<const char> in, const size_t expected_size, + const Ctx& ctx) +{ + int hash_size = FindNextChar(in, ')'); + if (hash_size < 1) return {}; + std::string val = std::string(in.begin(), in.begin() + hash_size); + if (!IsHex(val)) return {}; + auto hash = ParseHex(val); + if (hash.size() != expected_size) return {}; + return {{std::move(hash), hash_size}}; +} + +/** BuildBack pops the last two elements off `constructed` and wraps them in the specified Fragment */ +template<typename Key> +void BuildBack(Fragment nt, std::vector<NodeRef<Key>>& constructed, const bool reverse = false) +{ + NodeRef<Key> child = std::move(constructed.back()); + constructed.pop_back(); + if (reverse) { + constructed.back() = MakeNodeRef<Key>(nt, Vector(std::move(child), std::move(constructed.back()))); + } else { + constructed.back() = MakeNodeRef<Key>(nt, Vector(std::move(constructed.back()), std::move(child))); + } +} + +//! Parse a miniscript from its textual descriptor form. +template<typename Key, typename Ctx> +inline NodeRef<Key> Parse(Span<const char> in, const Ctx& ctx) +{ + using namespace spanparsing; + + // The two integers are used to hold state for thresh() + std::vector<std::tuple<ParseContext, int64_t, int64_t>> to_parse; + std::vector<NodeRef<Key>> constructed; + + to_parse.emplace_back(ParseContext::WRAPPED_EXPR, -1, -1); + + while (!to_parse.empty()) { + // Get the current context we are decoding within + auto [cur_context, n, k] = to_parse.back(); + to_parse.pop_back(); + + switch (cur_context) { + case ParseContext::WRAPPED_EXPR: { + int colon_index = -1; + for (int i = 1; i < (int)in.size(); ++i) { + if (in[i] == ':') { + colon_index = i; + break; + } + if (in[i] < 'a' || in[i] > 'z') break; + } + // If there is no colon, this loop won't execute + for (int j = 0; j < colon_index; ++j) { + if (in[j] == 'a') { + to_parse.emplace_back(ParseContext::ALT, -1, -1); + } else if (in[j] == 's') { + to_parse.emplace_back(ParseContext::SWAP, -1, -1); + } else if (in[j] == 'c') { + to_parse.emplace_back(ParseContext::CHECK, -1, -1); + } else if (in[j] == 'd') { + to_parse.emplace_back(ParseContext::DUP_IF, -1, -1); + } else if (in[j] == 'j') { + to_parse.emplace_back(ParseContext::NON_ZERO, -1, -1); + } else if (in[j] == 'n') { + to_parse.emplace_back(ParseContext::ZERO_NOTEQUAL, -1, -1); + } else if (in[j] == 'v') { + to_parse.emplace_back(ParseContext::VERIFY, -1, -1); + } else if (in[j] == 'u') { + to_parse.emplace_back(ParseContext::WRAP_U, -1, -1); + } else if (in[j] == 't') { + to_parse.emplace_back(ParseContext::WRAP_T, -1, -1); + } else if (in[j] == 'l') { + // The l: wrapper is equivalent to or_i(0,X) + constructed.push_back(MakeNodeRef<Key>(Fragment::JUST_0)); + to_parse.emplace_back(ParseContext::OR_I, -1, -1); + } else { + return {}; + } + } + to_parse.emplace_back(ParseContext::EXPR, -1, -1); + in = in.subspan(colon_index + 1); + break; + } + case ParseContext::EXPR: { + if (Const("0", in)) { + constructed.push_back(MakeNodeRef<Key>(Fragment::JUST_0)); + } else if (Const("1", in)) { + constructed.push_back(MakeNodeRef<Key>(Fragment::JUST_1)); + } else if (Const("pk(", in)) { + auto res = ParseKeyEnd<Key, Ctx>(in, ctx); + if (!res) return {}; + auto& [key, key_size] = *res; + constructed.push_back(MakeNodeRef<Key>(Fragment::WRAP_C, Vector(MakeNodeRef<Key>(Fragment::PK_K, Vector(std::move(key)))))); + in = in.subspan(key_size + 1); + } else if (Const("pkh(", in)) { + auto res = ParseKeyEnd<Key>(in, ctx); + if (!res) return {}; + auto& [key, key_size] = *res; + constructed.push_back(MakeNodeRef<Key>(Fragment::WRAP_C, Vector(MakeNodeRef<Key>(Fragment::PK_H, Vector(std::move(key)))))); + in = in.subspan(key_size + 1); + } else if (Const("pk_k(", in)) { + auto res = ParseKeyEnd<Key>(in, ctx); + if (!res) return {}; + auto& [key, key_size] = *res; + constructed.push_back(MakeNodeRef<Key>(Fragment::PK_K, Vector(std::move(key)))); + in = in.subspan(key_size + 1); + } else if (Const("pk_h(", in)) { + auto res = ParseKeyEnd<Key>(in, ctx); + if (!res) return {}; + auto& [key, key_size] = *res; + constructed.push_back(MakeNodeRef<Key>(Fragment::PK_H, Vector(std::move(key)))); + in = in.subspan(key_size + 1); + } else if (Const("sha256(", in)) { + auto res = ParseHexStrEnd(in, 32, ctx); + if (!res) return {}; + auto& [hash, hash_size] = *res; + constructed.push_back(MakeNodeRef<Key>(Fragment::SHA256, std::move(hash))); + in = in.subspan(hash_size + 1); + } else if (Const("ripemd160(", in)) { + auto res = ParseHexStrEnd(in, 20, ctx); + if (!res) return {}; + auto& [hash, hash_size] = *res; + constructed.push_back(MakeNodeRef<Key>(Fragment::RIPEMD160, std::move(hash))); + in = in.subspan(hash_size + 1); + } else if (Const("hash256(", in)) { + auto res = ParseHexStrEnd(in, 32, ctx); + if (!res) return {}; + auto& [hash, hash_size] = *res; + constructed.push_back(MakeNodeRef<Key>(Fragment::HASH256, std::move(hash))); + in = in.subspan(hash_size + 1); + } else if (Const("hash160(", in)) { + auto res = ParseHexStrEnd(in, 20, ctx); + if (!res) return {}; + auto& [hash, hash_size] = *res; + constructed.push_back(MakeNodeRef<Key>(Fragment::HASH160, std::move(hash))); + in = in.subspan(hash_size + 1); + } else if (Const("after(", in)) { + int arg_size = FindNextChar(in, ')'); + if (arg_size < 1) return {}; + int64_t num; + if (!ParseInt64(std::string(in.begin(), in.begin() + arg_size), &num)) return {}; + if (num < 1 || num >= 0x80000000L) return {}; + constructed.push_back(MakeNodeRef<Key>(Fragment::AFTER, num)); + in = in.subspan(arg_size + 1); + } else if (Const("older(", in)) { + int arg_size = FindNextChar(in, ')'); + if (arg_size < 1) return {}; + int64_t num; + if (!ParseInt64(std::string(in.begin(), in.begin() + arg_size), &num)) return {}; + if (num < 1 || num >= 0x80000000L) return {}; + constructed.push_back(MakeNodeRef<Key>(Fragment::OLDER, num)); + in = in.subspan(arg_size + 1); + } else if (Const("multi(", in)) { + // Get threshold + int next_comma = FindNextChar(in, ','); + if (next_comma < 1) return {}; + if (!ParseInt64(std::string(in.begin(), in.begin() + next_comma), &k)) return {}; + in = in.subspan(next_comma + 1); + // Get keys + std::vector<Key> keys; + while (next_comma != -1) { + Key key; + next_comma = FindNextChar(in, ','); + int key_length = (next_comma == -1) ? FindNextChar(in, ')') : next_comma; + if (key_length < 1) return {}; + if (!ctx.FromString(in.begin(), in.begin() + key_length, key)) return {}; + keys.push_back(std::move(key)); + in = in.subspan(key_length + 1); + } + if (keys.size() < 1 || keys.size() > 20) return {}; + if (k < 1 || k > (int64_t)keys.size()) return {}; + constructed.push_back(MakeNodeRef<Key>(Fragment::MULTI, std::move(keys), k)); + } else if (Const("thresh(", in)) { + int next_comma = FindNextChar(in, ','); + if (next_comma < 1) return {}; + if (!ParseInt64(std::string(in.begin(), in.begin() + next_comma), &k)) return {}; + if (k < 1) return {}; + in = in.subspan(next_comma + 1); + // n = 1 here because we read the first WRAPPED_EXPR before reaching THRESH + to_parse.emplace_back(ParseContext::THRESH, 1, k); + to_parse.emplace_back(ParseContext::WRAPPED_EXPR, -1, -1); + } else if (Const("andor(", in)) { + to_parse.emplace_back(ParseContext::ANDOR, -1, -1); + to_parse.emplace_back(ParseContext::CLOSE_BRACKET, -1, -1); + to_parse.emplace_back(ParseContext::WRAPPED_EXPR, -1, -1); + to_parse.emplace_back(ParseContext::COMMA, -1, -1); + to_parse.emplace_back(ParseContext::WRAPPED_EXPR, -1, -1); + to_parse.emplace_back(ParseContext::COMMA, -1, -1); + to_parse.emplace_back(ParseContext::WRAPPED_EXPR, -1, -1); + } else { + if (Const("and_n(", in)) { + to_parse.emplace_back(ParseContext::AND_N, -1, -1); + } else if (Const("and_b(", in)) { + to_parse.emplace_back(ParseContext::AND_B, -1, -1); + } else if (Const("and_v(", in)) { + to_parse.emplace_back(ParseContext::AND_V, -1, -1); + } else if (Const("or_b(", in)) { + to_parse.emplace_back(ParseContext::OR_B, -1, -1); + } else if (Const("or_c(", in)) { + to_parse.emplace_back(ParseContext::OR_C, -1, -1); + } else if (Const("or_d(", in)) { + to_parse.emplace_back(ParseContext::OR_D, -1, -1); + } else if (Const("or_i(", in)) { + to_parse.emplace_back(ParseContext::OR_I, -1, -1); + } else { + return {}; + } + to_parse.emplace_back(ParseContext::CLOSE_BRACKET, -1, -1); + to_parse.emplace_back(ParseContext::WRAPPED_EXPR, -1, -1); + to_parse.emplace_back(ParseContext::COMMA, -1, -1); + to_parse.emplace_back(ParseContext::WRAPPED_EXPR, -1, -1); + } + break; + } + case ParseContext::ALT: { + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_A, Vector(std::move(constructed.back()))); + break; + } + case ParseContext::SWAP: { + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_S, Vector(std::move(constructed.back()))); + break; + } + case ParseContext::CHECK: { + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_C, Vector(std::move(constructed.back()))); + break; + } + case ParseContext::DUP_IF: { + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_D, Vector(std::move(constructed.back()))); + break; + } + case ParseContext::NON_ZERO: { + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_J, Vector(std::move(constructed.back()))); + break; + } + case ParseContext::ZERO_NOTEQUAL: { + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_N, Vector(std::move(constructed.back()))); + break; + } + case ParseContext::VERIFY: { + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_V, Vector(std::move(constructed.back()))); + break; + } + case ParseContext::WRAP_U: { + constructed.back() = MakeNodeRef<Key>(Fragment::OR_I, Vector(std::move(constructed.back()), MakeNodeRef<Key>(Fragment::JUST_0))); + break; + } + case ParseContext::WRAP_T: { + constructed.back() = MakeNodeRef<Key>(Fragment::AND_V, Vector(std::move(constructed.back()), MakeNodeRef<Key>(Fragment::JUST_1))); + break; + } + case ParseContext::AND_B: { + BuildBack(Fragment::AND_B, constructed); + break; + } + case ParseContext::AND_N: { + auto mid = std::move(constructed.back()); + constructed.pop_back(); + constructed.back() = MakeNodeRef<Key>(Fragment::ANDOR, Vector(std::move(constructed.back()), std::move(mid), MakeNodeRef<Key>(Fragment::JUST_0))); + break; + } + case ParseContext::AND_V: { + BuildBack(Fragment::AND_V, constructed); + break; + } + case ParseContext::OR_B: { + BuildBack(Fragment::OR_B, constructed); + break; + } + case ParseContext::OR_C: { + BuildBack(Fragment::OR_C, constructed); + break; + } + case ParseContext::OR_D: { + BuildBack(Fragment::OR_D, constructed); + break; + } + case ParseContext::OR_I: { + BuildBack(Fragment::OR_I, constructed); + break; + } + case ParseContext::ANDOR: { + auto right = std::move(constructed.back()); + constructed.pop_back(); + auto mid = std::move(constructed.back()); + constructed.pop_back(); + constructed.back() = MakeNodeRef<Key>(Fragment::ANDOR, Vector(std::move(constructed.back()), std::move(mid), std::move(right))); + break; + } + case ParseContext::THRESH: { + if (in.size() < 1) return {}; + if (in[0] == ',') { + in = in.subspan(1); + to_parse.emplace_back(ParseContext::THRESH, n+1, k); + to_parse.emplace_back(ParseContext::WRAPPED_EXPR, -1, -1); + } else if (in[0] == ')') { + if (k > n) return {}; + in = in.subspan(1); + // Children are constructed in reverse order, so iterate from end to beginning + std::vector<NodeRef<Key>> subs; + for (int i = 0; i < n; ++i) { + subs.push_back(std::move(constructed.back())); + constructed.pop_back(); + } + std::reverse(subs.begin(), subs.end()); + constructed.push_back(MakeNodeRef<Key>(Fragment::THRESH, std::move(subs), k)); + } else { + return {}; + } + break; + } + case ParseContext::COMMA: { + if (in.size() < 1 || in[0] != ',') return {}; + in = in.subspan(1); + break; + } + case ParseContext::CLOSE_BRACKET: { + if (in.size() < 1 || in[0] != ')') return {}; + in = in.subspan(1); + break; + } + } + } + + // Sanity checks on the produced miniscript + assert(constructed.size() == 1); + if (in.size() > 0) return {}; + const NodeRef<Key> tl_node = std::move(constructed.front()); + if (!tl_node->IsValidTopLevel()) return {}; + return tl_node; +} + +/** Decode a script into opcode/push pairs. + * + * Construct a vector with one element per opcode in the script, in reverse order. + * Each element is a pair consisting of the opcode, as well as the data pushed by + * the opcode (including OP_n), if any. OP_CHECKSIGVERIFY, OP_CHECKMULTISIGVERIFY, + * and OP_EQUALVERIFY are decomposed into OP_CHECKSIG, OP_CHECKMULTISIG, OP_EQUAL + * respectively, plus OP_VERIFY. + */ +bool DecomposeScript(const CScript& script, std::vector<std::pair<opcodetype, std::vector<unsigned char>>>& out); + +/** Determine whether the passed pair (created by DecomposeScript) is pushing a number. */ +bool ParseScriptNumber(const std::pair<opcodetype, std::vector<unsigned char>>& in, int64_t& k); + +enum class DecodeContext { + /** A single expression of type B, K, or V. Specifically, this can't be an + * and_v or an expression of type W (a: and s: wrappers). */ + SINGLE_BKV_EXPR, + /** Potentially multiple SINGLE_BKV_EXPRs as children of (potentially multiple) + * and_v expressions. Syntactic sugar for MAYBE_AND_V + SINGLE_BKV_EXPR. */ + BKV_EXPR, + /** An expression of type W (a: or s: wrappers). */ + W_EXPR, + + /** SWAP expects the next element to be OP_SWAP (inside a W-type expression that + * didn't end with FROMALTSTACK), and wraps the top of the constructed stack + * with s: */ + SWAP, + /** ALT expects the next element to be TOALTSTACK (we must have already read a + * FROMALTSTACK earlier), and wraps the top of the constructed stack with a: */ + ALT, + /** CHECK wraps the top constructed node with c: */ + CHECK, + /** DUP_IF wraps the top constructed node with d: */ + DUP_IF, + /** VERIFY wraps the top constructed node with v: */ + VERIFY, + /** NON_ZERO wraps the top constructed node with j: */ + NON_ZERO, + /** ZERO_NOTEQUAL wraps the top constructed node with n: */ + ZERO_NOTEQUAL, + + /** MAYBE_AND_V will check if the next part of the script could be a valid + * miniscript sub-expression, and if so it will push AND_V and SINGLE_BKV_EXPR + * to decode it and construct the and_v node. This is recursive, to deal with + * multiple and_v nodes inside each other. */ + MAYBE_AND_V, + /** AND_V will construct an and_v node from the last two constructed nodes. */ + AND_V, + /** AND_B will construct an and_b node from the last two constructed nodes. */ + AND_B, + /** ANDOR will construct an andor node from the last three constructed nodes. */ + ANDOR, + /** OR_B will construct an or_b node from the last two constructed nodes. */ + OR_B, + /** OR_C will construct an or_c node from the last two constructed nodes. */ + OR_C, + /** OR_D will construct an or_d node from the last two constructed nodes. */ + OR_D, + + /** In a thresh expression, all sub-expressions other than the first are W-type, + * and end in OP_ADD. THRESH_W will check for this OP_ADD and either push a W_EXPR + * or a SINGLE_BKV_EXPR and jump to THRESH_E accordingly. */ + THRESH_W, + /** THRESH_E constructs a thresh node from the appropriate number of constructed + * children. */ + THRESH_E, + + /** ENDIF signals that we are inside some sort of OP_IF structure, which could be + * or_d, or_c, or_i, andor, d:, or j: wrapper, depending on what follows. We read + * a BKV_EXPR and then deal with the next opcode case-by-case. */ + ENDIF, + /** If, inside an ENDIF context, we find an OP_NOTIF before finding an OP_ELSE, + * we could either be in an or_d or an or_c node. We then check for IFDUP to + * distinguish these cases. */ + ENDIF_NOTIF, + /** If, inside an ENDIF context, we find an OP_ELSE, then we could be in either an + * or_i or an andor node. Read the next BKV_EXPR and find either an OP_IF or an + * OP_NOTIF. */ + ENDIF_ELSE, +}; + +//! Parse a miniscript from a bitcoin script +template<typename Key, typename Ctx, typename I> +inline NodeRef<Key> DecodeScript(I& in, I last, const Ctx& ctx) +{ + // The two integers are used to hold state for thresh() + std::vector<std::tuple<DecodeContext, int64_t, int64_t>> to_parse; + std::vector<NodeRef<Key>> constructed; + + // This is the top level, so we assume the type is B + // (in particular, disallowing top level W expressions) + to_parse.emplace_back(DecodeContext::BKV_EXPR, -1, -1); + + while (!to_parse.empty()) { + // Exit early if the Miniscript is not going to be valid. + if (!constructed.empty() && !constructed.back()->IsValid()) return {}; + + // Get the current context we are decoding within + auto [cur_context, n, k] = to_parse.back(); + to_parse.pop_back(); + + switch(cur_context) { + case DecodeContext::SINGLE_BKV_EXPR: { + if (in >= last) return {}; + + // Constants + if (in[0].first == OP_1) { + ++in; + constructed.push_back(MakeNodeRef<Key>(Fragment::JUST_1)); + break; + } + if (in[0].first == OP_0) { + ++in; + constructed.push_back(MakeNodeRef<Key>(Fragment::JUST_0)); + break; + } + // Public keys + if (in[0].second.size() == 33) { + Key key; + if (!ctx.FromPKBytes(in[0].second.begin(), in[0].second.end(), key)) return {}; + ++in; + constructed.push_back(MakeNodeRef<Key>(Fragment::PK_K, Vector(std::move(key)))); + break; + } + if (last - in >= 5 && in[0].first == OP_VERIFY && in[1].first == OP_EQUAL && in[3].first == OP_HASH160 && in[4].first == OP_DUP && in[2].second.size() == 20) { + Key key; + if (!ctx.FromPKHBytes(in[2].second.begin(), in[2].second.end(), key)) return {}; + in += 5; + constructed.push_back(MakeNodeRef<Key>(Fragment::PK_H, Vector(std::move(key)))); + break; + } + // Time locks + if (last - in >= 2 && in[0].first == OP_CHECKSEQUENCEVERIFY && ParseScriptNumber(in[1], k)) { + in += 2; + if (k < 1 || k > 0x7FFFFFFFL) return {}; + constructed.push_back(MakeNodeRef<Key>(Fragment::OLDER, k)); + break; + } + if (last - in >= 2 && in[0].first == OP_CHECKLOCKTIMEVERIFY && ParseScriptNumber(in[1], k)) { + in += 2; + if (k < 1 || k > 0x7FFFFFFFL) return {}; + constructed.push_back(MakeNodeRef<Key>(Fragment::AFTER, k)); + break; + } + // Hashes + if (last - in >= 7 && in[0].first == OP_EQUAL && in[3].first == OP_VERIFY && in[4].first == OP_EQUAL && ParseScriptNumber(in[5], k) && k == 32 && in[6].first == OP_SIZE) { + if (in[2].first == OP_SHA256 && in[1].second.size() == 32) { + constructed.push_back(MakeNodeRef<Key>(Fragment::SHA256, in[1].second)); + in += 7; + break; + } else if (in[2].first == OP_RIPEMD160 && in[1].second.size() == 20) { + constructed.push_back(MakeNodeRef<Key>(Fragment::RIPEMD160, in[1].second)); + in += 7; + break; + } else if (in[2].first == OP_HASH256 && in[1].second.size() == 32) { + constructed.push_back(MakeNodeRef<Key>(Fragment::HASH256, in[1].second)); + in += 7; + break; + } else if (in[2].first == OP_HASH160 && in[1].second.size() == 20) { + constructed.push_back(MakeNodeRef<Key>(Fragment::HASH160, in[1].second)); + in += 7; + break; + } + } + // Multi + if (last - in >= 3 && in[0].first == OP_CHECKMULTISIG) { + std::vector<Key> keys; + if (!ParseScriptNumber(in[1], n)) return {}; + if (last - in < 3 + n) return {}; + if (n < 1 || n > 20) return {}; + for (int i = 0; i < n; ++i) { + Key key; + if (in[2 + i].second.size() != 33) return {}; + if (!ctx.FromPKBytes(in[2 + i].second.begin(), in[2 + i].second.end(), key)) return {}; + keys.push_back(std::move(key)); + } + if (!ParseScriptNumber(in[2 + n], k)) return {}; + if (k < 1 || k > n) return {}; + in += 3 + n; + std::reverse(keys.begin(), keys.end()); + constructed.push_back(MakeNodeRef<Key>(Fragment::MULTI, std::move(keys), k)); + break; + } + /** In the following wrappers, we only need to push SINGLE_BKV_EXPR rather + * than BKV_EXPR, because and_v commutes with these wrappers. For example, + * c:and_v(X,Y) produces the same script as and_v(X,c:Y). */ + // c: wrapper + if (in[0].first == OP_CHECKSIG) { + ++in; + to_parse.emplace_back(DecodeContext::CHECK, -1, -1); + to_parse.emplace_back(DecodeContext::SINGLE_BKV_EXPR, -1, -1); + break; + } + // v: wrapper + if (in[0].first == OP_VERIFY) { + ++in; + to_parse.emplace_back(DecodeContext::VERIFY, -1, -1); + to_parse.emplace_back(DecodeContext::SINGLE_BKV_EXPR, -1, -1); + break; + } + // n: wrapper + if (in[0].first == OP_0NOTEQUAL) { + ++in; + to_parse.emplace_back(DecodeContext::ZERO_NOTEQUAL, -1, -1); + to_parse.emplace_back(DecodeContext::SINGLE_BKV_EXPR, -1, -1); + break; + } + // Thresh + if (last - in >= 3 && in[0].first == OP_EQUAL && ParseScriptNumber(in[1], k)) { + if (k < 1) return {}; + in += 2; + to_parse.emplace_back(DecodeContext::THRESH_W, 0, k); + break; + } + // OP_ENDIF can be WRAP_J, WRAP_D, ANDOR, OR_C, OR_D, or OR_I + if (in[0].first == OP_ENDIF) { + ++in; + to_parse.emplace_back(DecodeContext::ENDIF, -1, -1); + to_parse.emplace_back(DecodeContext::BKV_EXPR, -1, -1); + break; + } + /** In and_b and or_b nodes, we only look for SINGLE_BKV_EXPR, because + * or_b(and_v(X,Y),Z) has script [X] [Y] [Z] OP_BOOLOR, the same as + * and_v(X,or_b(Y,Z)). In this example, the former of these is invalid as + * miniscript, while the latter is valid. So we leave the and_v "outside" + * while decoding. */ + // and_b + if (in[0].first == OP_BOOLAND) { + ++in; + to_parse.emplace_back(DecodeContext::AND_B, -1, -1); + to_parse.emplace_back(DecodeContext::SINGLE_BKV_EXPR, -1, -1); + to_parse.emplace_back(DecodeContext::W_EXPR, -1, -1); + break; + } + // or_b + if (in[0].first == OP_BOOLOR) { + ++in; + to_parse.emplace_back(DecodeContext::OR_B, -1, -1); + to_parse.emplace_back(DecodeContext::SINGLE_BKV_EXPR, -1, -1); + to_parse.emplace_back(DecodeContext::W_EXPR, -1, -1); + break; + } + // Unrecognised expression + return {}; + } + case DecodeContext::BKV_EXPR: { + to_parse.emplace_back(DecodeContext::MAYBE_AND_V, -1, -1); + to_parse.emplace_back(DecodeContext::SINGLE_BKV_EXPR, -1, -1); + break; + } + case DecodeContext::W_EXPR: { + // a: wrapper + if (in >= last) return {}; + if (in[0].first == OP_FROMALTSTACK) { + ++in; + to_parse.emplace_back(DecodeContext::ALT, -1, -1); + } else { + to_parse.emplace_back(DecodeContext::SWAP, -1, -1); + } + to_parse.emplace_back(DecodeContext::BKV_EXPR, -1, -1); + break; + } + case DecodeContext::MAYBE_AND_V: { + // If we reach a potential AND_V top-level, check if the next part of the script could be another AND_V child + // These op-codes cannot end any well-formed miniscript so cannot be used in an and_v node. + if (in < last && in[0].first != OP_IF && in[0].first != OP_ELSE && in[0].first != OP_NOTIF && in[0].first != OP_TOALTSTACK && in[0].first != OP_SWAP) { + to_parse.emplace_back(DecodeContext::AND_V, -1, -1); + // BKV_EXPR can contain more AND_V nodes + to_parse.emplace_back(DecodeContext::BKV_EXPR, -1, -1); + } + break; + } + case DecodeContext::SWAP: { + if (in >= last || in[0].first != OP_SWAP || constructed.empty()) return {}; + ++in; + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_S, Vector(std::move(constructed.back()))); + break; + } + case DecodeContext::ALT: { + if (in >= last || in[0].first != OP_TOALTSTACK || constructed.empty()) return {}; + ++in; + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_A, Vector(std::move(constructed.back()))); + break; + } + case DecodeContext::CHECK: { + if (constructed.empty()) return {}; + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_C, Vector(std::move(constructed.back()))); + break; + } + case DecodeContext::DUP_IF: { + if (constructed.empty()) return {}; + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_D, Vector(std::move(constructed.back()))); + break; + } + case DecodeContext::VERIFY: { + if (constructed.empty()) return {}; + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_V, Vector(std::move(constructed.back()))); + break; + } + case DecodeContext::NON_ZERO: { + if (constructed.empty()) return {}; + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_J, Vector(std::move(constructed.back()))); + break; + } + case DecodeContext::ZERO_NOTEQUAL: { + if (constructed.empty()) return {}; + constructed.back() = MakeNodeRef<Key>(Fragment::WRAP_N, Vector(std::move(constructed.back()))); + break; + } + case DecodeContext::AND_V: { + if (constructed.size() < 2) return {}; + BuildBack(Fragment::AND_V, constructed, /* reverse */ true); + break; + } + case DecodeContext::AND_B: { + if (constructed.size() < 2) return {}; + BuildBack(Fragment::AND_B, constructed, /* reverse */ true); + break; + } + case DecodeContext::OR_B: { + if (constructed.size() < 2) return {}; + BuildBack(Fragment::OR_B, constructed, /* reverse */ true); + break; + } + case DecodeContext::OR_C: { + if (constructed.size() < 2) return {}; + BuildBack(Fragment::OR_C, constructed, /* reverse */ true); + break; + } + case DecodeContext::OR_D: { + if (constructed.size() < 2) return {}; + BuildBack(Fragment::OR_D, constructed, /* reverse */ true); + break; + } + case DecodeContext::ANDOR: { + if (constructed.size() < 3) return {}; + NodeRef<Key> left = std::move(constructed.back()); + constructed.pop_back(); + NodeRef<Key> right = std::move(constructed.back()); + constructed.pop_back(); + NodeRef<Key> mid = std::move(constructed.back()); + constructed.back() = MakeNodeRef<Key>(Fragment::ANDOR, Vector(std::move(left), std::move(mid), std::move(right))); + break; + } + case DecodeContext::THRESH_W: { + if (in >= last) return {}; + if (in[0].first == OP_ADD) { + ++in; + to_parse.emplace_back(DecodeContext::THRESH_W, n+1, k); + to_parse.emplace_back(DecodeContext::W_EXPR, -1, -1); + } else { + to_parse.emplace_back(DecodeContext::THRESH_E, n+1, k); + // All children of thresh have type modifier d, so cannot be and_v + to_parse.emplace_back(DecodeContext::SINGLE_BKV_EXPR, -1, -1); + } + break; + } + case DecodeContext::THRESH_E: { + if (k < 1 || k > n || constructed.size() < static_cast<size_t>(n)) return {}; + std::vector<NodeRef<Key>> subs; + for (int i = 0; i < n; ++i) { + NodeRef<Key> sub = std::move(constructed.back()); + constructed.pop_back(); + subs.push_back(std::move(sub)); + } + constructed.push_back(MakeNodeRef<Key>(Fragment::THRESH, std::move(subs), k)); + break; + } + case DecodeContext::ENDIF: { + if (in >= last) return {}; + + // could be andor or or_i + if (in[0].first == OP_ELSE) { + ++in; + to_parse.emplace_back(DecodeContext::ENDIF_ELSE, -1, -1); + to_parse.emplace_back(DecodeContext::BKV_EXPR, -1, -1); + } + // could be j: or d: wrapper + else if (in[0].first == OP_IF) { + if (last - in >= 2 && in[1].first == OP_DUP) { + in += 2; + to_parse.emplace_back(DecodeContext::DUP_IF, -1, -1); + } else if (last - in >= 3 && in[1].first == OP_0NOTEQUAL && in[2].first == OP_SIZE) { + in += 3; + to_parse.emplace_back(DecodeContext::NON_ZERO, -1, -1); + } + else { + return {}; + } + // could be or_c or or_d + } else if (in[0].first == OP_NOTIF) { + ++in; + to_parse.emplace_back(DecodeContext::ENDIF_NOTIF, -1, -1); + } + else { + return {}; + } + break; + } + case DecodeContext::ENDIF_NOTIF: { + if (in >= last) return {}; + if (in[0].first == OP_IFDUP) { + ++in; + to_parse.emplace_back(DecodeContext::OR_D, -1, -1); + } else { + to_parse.emplace_back(DecodeContext::OR_C, -1, -1); + } + // or_c and or_d both require X to have type modifier d so, can't contain and_v + to_parse.emplace_back(DecodeContext::SINGLE_BKV_EXPR, -1, -1); + break; + } + case DecodeContext::ENDIF_ELSE: { + if (in >= last) return {}; + if (in[0].first == OP_IF) { + ++in; + BuildBack(Fragment::OR_I, constructed, /* reverse */ true); + } else if (in[0].first == OP_NOTIF) { + ++in; + to_parse.emplace_back(DecodeContext::ANDOR, -1, -1); + // andor requires X to have type modifier d, so it can't be and_v + to_parse.emplace_back(DecodeContext::SINGLE_BKV_EXPR, -1, -1); + } else { + return {}; + } + break; + } + } + } + if (constructed.size() != 1) return {}; + const NodeRef<Key> tl_node = std::move(constructed.front()); + // Note that due to how ComputeType works (only assign the type to the node if the + // subs' types are valid) this would fail if any node of tree is badly typed. + if (!tl_node->IsValidTopLevel()) return {}; + return tl_node; +} + +} // namespace internal + +template<typename Ctx> +inline NodeRef<typename Ctx::Key> FromString(const std::string& str, const Ctx& ctx) { + return internal::Parse<typename Ctx::Key>(str, ctx); +} + +template<typename Ctx> +inline NodeRef<typename Ctx::Key> FromScript(const CScript& script, const Ctx& ctx) { + using namespace internal; + std::vector<std::pair<opcodetype, std::vector<unsigned char>>> decomposed; + if (!DecomposeScript(script, decomposed)) return {}; + auto it = decomposed.begin(); + auto ret = DecodeScript<typename Ctx::Key>(it, decomposed.end(), ctx); + if (!ret) return {}; + if (it != decomposed.end()) return {}; + return ret; +} + +} // namespace miniscript + +#endif // BITCOIN_SCRIPT_MINISCRIPT_H diff --git a/src/script/script.cpp b/src/script/script.cpp index 9a6419088b..88b4bc2f44 100644 --- a/src/script/script.cpp +++ b/src/script/script.cpp @@ -339,3 +339,28 @@ bool IsOpSuccess(const opcodetype& opcode) (opcode >= 141 && opcode <= 142) || (opcode >= 149 && opcode <= 153) || (opcode >= 187 && opcode <= 254); } + +bool CheckMinimalPush(const std::vector<unsigned char>& data, opcodetype opcode) { + // Excludes OP_1NEGATE, OP_1-16 since they are by definition minimal + assert(0 <= opcode && opcode <= OP_PUSHDATA4); + if (data.size() == 0) { + // Should have used OP_0. + return opcode == OP_0; + } else if (data.size() == 1 && data[0] >= 1 && data[0] <= 16) { + // Should have used OP_1 .. OP_16. + return false; + } else if (data.size() == 1 && data[0] == 0x81) { + // Should have used OP_1NEGATE. + return false; + } else if (data.size() <= 75) { + // Must have used a direct push (opcode indicating number of bytes pushed + those bytes). + return opcode == data.size(); + } else if (data.size() <= 255) { + // Must have used OP_PUSHDATA. + return opcode == OP_PUSHDATA1; + } else if (data.size() <= 65535) { + // Must have used OP_PUSHDATA2. + return opcode == OP_PUSHDATA2; + } + return true; +} diff --git a/src/script/script.h b/src/script/script.h index a89c987306..3b799ad637 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -335,6 +335,8 @@ public: return m_value; } + int64_t GetInt64() const { return m_value; } + std::vector<unsigned char> getvch() const { return serialize(m_value); @@ -576,4 +578,31 @@ struct CScriptWitness /** Test for OP_SUCCESSx opcodes as defined by BIP342. */ bool IsOpSuccess(const opcodetype& opcode); +bool CheckMinimalPush(const std::vector<unsigned char>& data, opcodetype opcode); + +/** Build a script by concatenating other scripts, or any argument accepted by CScript::operator<<. */ +template<typename... Ts> +CScript BuildScript(Ts&&... inputs) +{ + CScript ret; + int cnt{0}; + + ([&ret, &cnt] (Ts&& input) { + cnt++; + if constexpr (std::is_same_v<std::remove_cv_t<std::remove_reference_t<Ts>>, CScript>) { + // If it is a CScript, extend ret with it. Move or copy the first element instead. + if (cnt == 0) { + ret = std::forward<Ts>(input); + } else { + ret.insert(ret.end(), input.begin(), input.end()); + } + } else { + // Otherwise invoke CScript::operator<<. + ret << input; + } + } (std::forward<Ts>(inputs)), ...); + + return ret; +} + #endif // BITCOIN_SCRIPT_SCRIPT_H diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 2e5c49e0b6..d77515f16c 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -655,7 +655,7 @@ bool SignTransaction(CMutableTransaction& mtx, const SigningProvider* keystore, CTxIn& txin = mtx.vin[i]; auto coin = coins.find(txin.prevout); if (coin == coins.end() || coin->second.IsSpent()) { - txdata.Init(txConst, /* spent_outputs */ {}, /* force */ true); + txdata.Init(txConst, /*spent_outputs=*/{}, /*force=*/true); break; } else { spent_outputs.emplace_back(coin->second.out.nValue, coin->second.out.scriptPubKey); diff --git a/src/script/standard.cpp b/src/script/standard.cpp index 806b3169cd..e25155d3dd 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -91,11 +91,6 @@ static constexpr bool IsSmallInteger(opcodetype opcode) return opcode >= OP_1 && opcode <= OP_16; } -static constexpr bool IsPushdataOp(opcodetype opcode) -{ - return opcode > OP_FALSE && opcode <= OP_PUSHDATA4; -} - /** Retrieve a minimally-encoded number in range [min,max] from an (opcode, data) pair, * whether it's OP_n or through a push. */ static std::optional<int> GetScriptNumber(opcodetype opcode, valtype data, int min, int max) @@ -398,13 +393,7 @@ void TaprootSpendData::Merge(TaprootSpendData other) merkle_root = other.merkle_root; } for (auto& [key, control_blocks] : other.scripts) { - // Once P0083R3 is supported by all our targeted platforms, - // this loop body can be replaced with: - // scripts[key].merge(std::move(control_blocks)); - auto& target = scripts[key]; - for (auto& control_block: control_blocks) { - target.insert(std::move(control_block)); - } + scripts[key].merge(std::move(control_blocks)); } } diff --git a/src/script/standard.h b/src/script/standard.h index 75bfe2db38..f0b143c52b 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -162,6 +162,11 @@ bool IsValidDestination(const CTxDestination& dest); /** Get the name of a TxoutType as a string */ std::string GetTxnOutputType(TxoutType t); +constexpr bool IsPushdataOp(opcodetype opcode) +{ + return opcode > OP_FALSE && opcode <= OP_PUSHDATA4; +} + /** * Parse a scriptPubKey and identify script type for standard scripts. If * successful, returns script type and parsed pubkeys or hashes, depending on diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp index 6965f40253..ea1a27c6f6 100644 --- a/src/support/lockedpool.cpp +++ b/src/support/lockedpool.cpp @@ -235,12 +235,6 @@ PosixLockedPageAllocator::PosixLockedPageAllocator() #endif } -// Some systems (at least OS X) do not define MAP_ANONYMOUS yet and define -// MAP_ANON which is deprecated -#ifndef MAP_ANONYMOUS -#define MAP_ANONYMOUS MAP_ANON -#endif - void *PosixLockedPageAllocator::AllocateLocked(size_t len, bool *lockingSuccess) { void *addr; diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index a17cc87730..f03ff5ba3a 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -34,8 +34,6 @@ static CService ip(uint32_t i) return CService(CNetAddr(s), Params().GetDefaultPort()); } -static NodeId id = 0; - void UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_seconds); BOOST_FIXTURE_TEST_SUITE(denialofservice_tests, TestingSetup) @@ -59,6 +57,7 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) // Mock an outbound peer CAddress addr1(ip(0xa0b0c001), NODE_NONE); + NodeId id{0}; CNode dummyNode1{id++, ServiceFlags(NODE_NETWORK | NODE_WITNESS), /*sock=*/nullptr, @@ -114,7 +113,7 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) peerLogic->FinalizeNode(dummyNode1); } -static void AddRandomOutboundPeer(std::vector<CNode*>& vNodes, PeerManager& peerLogic, ConnmanTestMsg& connman, ConnectionType connType) +static void AddRandomOutboundPeer(NodeId& id, std::vector<CNode*>& vNodes, PeerManager& peerLogic, ConnmanTestMsg& connman, ConnectionType connType) { CAddress addr(ip(g_insecure_rand_ctx.randbits(32)), NODE_NONE); vNodes.emplace_back(new CNode{id++, @@ -138,6 +137,7 @@ static void AddRandomOutboundPeer(std::vector<CNode*>& vNodes, PeerManager& peer BOOST_AUTO_TEST_CASE(stale_tip_peer_management) { + NodeId id{0}; const CChainParams& chainparams = Params(); auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman); auto peerLogic = PeerManager::make(chainparams, *connman, *m_node.addrman, nullptr, @@ -157,7 +157,7 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) // Mock some outbound peers for (int i = 0; i < max_outbound_full_relay; ++i) { - AddRandomOutboundPeer(vNodes, *peerLogic, *connman, ConnectionType::OUTBOUND_FULL_RELAY); + AddRandomOutboundPeer(id, vNodes, *peerLogic, *connman, ConnectionType::OUTBOUND_FULL_RELAY); } peerLogic->CheckForStaleTipAndEvictPeers(); @@ -183,7 +183,7 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) // on the next check (since we're mocking the time to be in the future, the // required time connected check should be satisfied). SetMockTime(time_init); - AddRandomOutboundPeer(vNodes, *peerLogic, *connman, ConnectionType::OUTBOUND_FULL_RELAY); + AddRandomOutboundPeer(id, vNodes, *peerLogic, *connman, ConnectionType::OUTBOUND_FULL_RELAY); SetMockTime(time_later); peerLogic->CheckForStaleTipAndEvictPeers(); @@ -215,6 +215,7 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) BOOST_AUTO_TEST_CASE(block_relay_only_eviction) { + NodeId id{0}; const CChainParams& chainparams = Params(); auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman); auto peerLogic = PeerManager::make(chainparams, *connman, *m_node.addrman, nullptr, @@ -232,7 +233,7 @@ BOOST_AUTO_TEST_CASE(block_relay_only_eviction) // Add block-relay-only peers up to the limit for (int i = 0; i < max_outbound_block_relay; ++i) { - AddRandomOutboundPeer(vNodes, *peerLogic, *connman, ConnectionType::BLOCK_RELAY); + AddRandomOutboundPeer(id, vNodes, *peerLogic, *connman, ConnectionType::BLOCK_RELAY); } peerLogic->CheckForStaleTipAndEvictPeers(); @@ -241,7 +242,7 @@ BOOST_AUTO_TEST_CASE(block_relay_only_eviction) } // Add an extra block-relay-only peer breaking the limit (mocks logic in ThreadOpenConnections) - AddRandomOutboundPeer(vNodes, *peerLogic, *connman, ConnectionType::BLOCK_RELAY); + AddRandomOutboundPeer(id, vNodes, *peerLogic, *connman, ConnectionType::BLOCK_RELAY); peerLogic->CheckForStaleTipAndEvictPeers(); // The extra peer should only get marked for eviction after MINIMUM_CONNECT_TIME @@ -297,6 +298,7 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) std::array<CNode*, 3> nodes; banman->ClearBanned(); + NodeId id{0}; nodes[0] = new CNode{id++, NODE_NETWORK, /*sock=*/nullptr, @@ -403,6 +405,7 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) SetMockTime(nStartTime); // Overrides future calls to GetTime() CAddress addr(ip(0xa0b0c001), NODE_NONE); + NodeId id{0}; CNode dummyNode{id++, NODE_NETWORK, /*sock=*/nullptr, diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 55a9a78159..30add9c16d 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -311,7 +311,7 @@ void DoCheck(const std::string& prv, const std::string& pub, const std::string& spend.vout.resize(1); std::vector<CTxOut> utxos(1); PrecomputedTransactionData txdata; - txdata.Init(spend, std::move(utxos), /* force */ true); + txdata.Init(spend, std::move(utxos), /*force=*/true); MutableTransactionSignatureCreator creator(&spend, 0, CAmount{0}, &txdata, SIGHASH_DEFAULT); SignatureData sigdata; BOOST_CHECK_MESSAGE(ProduceSignature(Merge(keys_priv, script_provider), creator, spks[n], sigdata), prv); @@ -381,17 +381,17 @@ BOOST_AUTO_TEST_CASE(descriptor_test) Check("wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"00149a1c78a507689f6f54b847ad1cef1e614ee23f1e"}}, OutputType::BECH32); Check("sh(wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "sh(wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", "sh(wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "sh(wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", SIGNABLE, {{"a91484ab21b1b2fd065d4504ff693d832434b6108d7b87"}}, OutputType::P2SH_SEGWIT); Check("tr(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "tr(a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "tr(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "tr(a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE | XONLY_KEYS, {{"512077aab6e066f8a7419c5ab714c12c67d25007ed55a43cadcacb4d7a970a093f11"}}, OutputType::BECH32M); - CheckUnparsable("sh(wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY2))", "sh(wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5))", "Pubkey '03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5' is invalid"); // Invalid pubkey - CheckUnparsable("pkh(deadbeef/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh(deadbeef/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "Key origin start '[ character expected but not found, got 'd' instead"); // Missing start bracket in key origin - CheckUnparsable("pkh([deadbeef]/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh([deadbeef]/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "Multiple ']' characters found for a single pubkey"); // Multiple end brackets in key origin + CheckUnparsable("sh(wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY2))", "sh(wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5))", "wpkh(): Pubkey '03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5' is invalid"); // Invalid pubkey + CheckUnparsable("pkh(deadbeef/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh(deadbeef/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "pkh(): Key origin start '[ character expected but not found, got 'd' instead"); // Missing start bracket in key origin + CheckUnparsable("pkh([deadbeef]/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh([deadbeef]/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "pkh(): Multiple ']' characters found for a single pubkey"); // Multiple end brackets in key origin // Basic single-key uncompressed Check("combo(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "combo(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "combo(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "combo(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)",SIGNABLE, {{"4104a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235ac","76a914b5bd079c4d57cc7fc28ecf8213a6b791625b818388ac"}}, std::nullopt); Check("pk(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "pk(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "pk(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "pk(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", SIGNABLE, {{"4104a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235ac"}}, std::nullopt); Check("pkh(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "pkh(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "pkh(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "pkh(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", SIGNABLE, {{"76a914b5bd079c4d57cc7fc28ecf8213a6b791625b818388ac"}}, OutputType::LEGACY); - CheckUnparsable("wpkh(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "wpkh(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "Uncompressed keys are not allowed"); // No uncompressed keys in witness - CheckUnparsable("wsh(pk(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss))", "wsh(pk(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235))", "Uncompressed keys are not allowed"); // No uncompressed keys in witness - CheckUnparsable("sh(wpkh(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss))", "sh(wpkh(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235))", "Uncompressed keys are not allowed"); // No uncompressed keys in witness + CheckUnparsable("wpkh(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "wpkh(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "wpkh(): Uncompressed keys are not allowed"); // No uncompressed keys in witness + CheckUnparsable("wsh(pk(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss))", "wsh(pk(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235))", "pk(): Uncompressed keys are not allowed"); // No uncompressed keys in witness + CheckUnparsable("sh(wpkh(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss))", "sh(wpkh(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235))", "wpkh(): Uncompressed keys are not allowed"); // No uncompressed keys in witness // Some unconventional single-key constructions Check("sh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "sh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", "sh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "sh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", SIGNABLE, {{"a9141857af51a5e516552b3086430fd8ce55f7c1a52487"}}, OutputType::LEGACY); @@ -415,9 +415,9 @@ BOOST_AUTO_TEST_CASE(descriptor_test) // Mixed range xpubs and const pubkeys Check("multi(1,xprvA2JDeKCSNNZky6uBCviVfJSKyQ1mDYahRjijr5idH2WwLsEd4Hsb2Tyh8RfQMuPh7f7RtyzTtdrbdqqsunu5Mm3wDvUAKRHSC34sJ7in334/*,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)","multi(1,xpub6FHa3pjLCk84BayeJxFW2SP4XRrFd1JYnxeLeU8EqN3vDfZmbqBqaGJAyiLjTAwm6ZLRQUMv1ZACTj37sR62cfN7fe5JnJ7dh8zL4fiyLHV/*,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)","multi(1,xprvA2JDeKCSNNZky6uBCviVfJSKyQ1mDYahRjijr5idH2WwLsEd4Hsb2Tyh8RfQMuPh7f7RtyzTtdrbdqqsunu5Mm3wDvUAKRHSC34sJ7in334/*,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)","multi(1,xpub6FHa3pjLCk84BayeJxFW2SP4XRrFd1JYnxeLeU8EqN3vDfZmbqBqaGJAyiLjTAwm6ZLRQUMv1ZACTj37sR62cfN7fe5JnJ7dh8zL4fiyLHV/*,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", RANGE | MIXED_PUBKEYS, {{"512102df12b7035bdac8e3bab862a3a83d06ea6b17b6753d52edecba9be46f5d09e0762103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd52ae"},{"5121032869a233c9adff9a994e4966e5b821fd5bac066da6c3112488dc52383b4a98ec2103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd52ae"},{"5121035d30b6c66dc1e036c45369da8287518cf7e0d6ed1e2b905171c605708f14ca032103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd52ae"}}, std::nullopt,{{2},{1},{0},{}}); - CheckUnparsable("combo([012345678]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc)", "combo([012345678]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL)", "Fingerprint is not 4 bytes (9 characters instead of 8 characters)"); // Too long key fingerprint - CheckUnparsable("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483648)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483648)", "Key path value 2147483648 is out of range"); // BIP 32 path element overflow - CheckUnparsable("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/1aa)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/1aa)", "Key path value '1aa' is not a valid uint32"); // Path is not valid uint + CheckUnparsable("combo([012345678]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc)", "combo([012345678]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL)", "combo(): Fingerprint is not 4 bytes (9 characters instead of 8 characters)"); // Too long key fingerprint + CheckUnparsable("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483648)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483648)", "pkh(): Key path value 2147483648 is out of range"); // BIP 32 path element overflow + CheckUnparsable("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/1aa)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/1aa)", "pkh(): Key path value '1aa' is not a valid uint32"); // Path is not valid uint Check("pkh([01234567/10/20]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0)", "pkh([01234567/10/20]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0)", "pkh([01234567/10/20/2147483647']xprv9vHkqa6XAPwKqSKSEJMcAB3yoCZhaSVsGZbSkFY5L3Lfjjk8sjZucbsbvEw5o3QrSA69nPfZDCgFnNnLhQ2ohpZuwummndnPasDw2Qr6dC2/0)", "pkh([01234567/10/20/2147483647']xpub69H7F5dQzmVd3vPuLKtcXJziMEQByuDidnX3YdwgtNsecY5HRGtAAQC5mXTt4dsv9RzyjgDjAQs9VGVV6ydYCHnprc9vvaA5YtqWyL6hyds/0)", HARDENED, {{"76a914ebdc90806a9c4356c1c88e42216611e1cb4c1c1788ac"}}, OutputType::LEGACY, {{10, 20, 0xFFFFFFFFUL, 0}}); // Multisig constructions @@ -430,11 +430,11 @@ BOOST_AUTO_TEST_CASE(descriptor_test) Check("sh(wsh(multi(16,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGNz6YCCQtYvFzMtrC6D3tKTKdBBboMrLTsjr2NYVBwapCkn7Mr,KxogYhiNfwxuswvXV66eFyKcCpm7dZ7TqHVqujHAVUjJxyivxQ9X,L2BUNduTSyZwZjwNHynQTF14mv2uz2NRq5n5sYWTb4FkkmqgEE9f,L1okJGHGn1kFjdXHKxXjwVVtmCMR2JA5QsbKCSpSb7ReQjezKeoD,KxDCNSST75HFPaW5QKpzHtAyaCQC7p9Vo3FYfi2u4dXD1vgMiboK,L5edQjFtnkcf5UWURn6UuuoFrabgDQUHdheKCziwN42aLwS3KizU,KzF8UWFcEC7BYTq8Go1xVimMkDmyNYVmXV5PV7RuDicvAocoPB8i,L3nHUboKG2w4VSJ5jYZ5CBM97oeK6YuKvfZxrefdShECcjEYKMWZ,KyjHo36dWkYhimKmVVmQTq3gERv3pnqA4xFCpvUgbGDJad7eS8WE,KwsfyHKRUTZPQtysN7M3tZ4GXTnuov5XRgjdF2XCG8faAPmFruRF,KzCUbGhN9LJhdeFfL9zQgTJMjqxdBKEekRGZX24hXdgCNCijkkap,KzgpMBwwsDLwkaC5UrmBgCYaBD2WgZ7PBoGYXR8KT7gCA9UTN5a3,KyBXTPy4T7YG4q9tcAM3LkvfRpD1ybHMvcJ2ehaWXaSqeGUxEdkP,KzJDe9iwJRPtKP2F2AoN6zBgzS7uiuAwhWCfGdNeYJ3PC1HNJ8M8,L1xbHrxynrqLKkoYc4qtoQPx6uy5qYXR5ZDYVYBSRmCV5piU3JG9)))","sh(wsh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232)))", "sh(wsh(multi(16,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGNz6YCCQtYvFzMtrC6D3tKTKdBBboMrLTsjr2NYVBwapCkn7Mr,KxogYhiNfwxuswvXV66eFyKcCpm7dZ7TqHVqujHAVUjJxyivxQ9X,L2BUNduTSyZwZjwNHynQTF14mv2uz2NRq5n5sYWTb4FkkmqgEE9f,L1okJGHGn1kFjdXHKxXjwVVtmCMR2JA5QsbKCSpSb7ReQjezKeoD,KxDCNSST75HFPaW5QKpzHtAyaCQC7p9Vo3FYfi2u4dXD1vgMiboK,L5edQjFtnkcf5UWURn6UuuoFrabgDQUHdheKCziwN42aLwS3KizU,KzF8UWFcEC7BYTq8Go1xVimMkDmyNYVmXV5PV7RuDicvAocoPB8i,L3nHUboKG2w4VSJ5jYZ5CBM97oeK6YuKvfZxrefdShECcjEYKMWZ,KyjHo36dWkYhimKmVVmQTq3gERv3pnqA4xFCpvUgbGDJad7eS8WE,KwsfyHKRUTZPQtysN7M3tZ4GXTnuov5XRgjdF2XCG8faAPmFruRF,KzCUbGhN9LJhdeFfL9zQgTJMjqxdBKEekRGZX24hXdgCNCijkkap,KzgpMBwwsDLwkaC5UrmBgCYaBD2WgZ7PBoGYXR8KT7gCA9UTN5a3,KyBXTPy4T7YG4q9tcAM3LkvfRpD1ybHMvcJ2ehaWXaSqeGUxEdkP,KzJDe9iwJRPtKP2F2AoN6zBgzS7uiuAwhWCfGdNeYJ3PC1HNJ8M8,L1xbHrxynrqLKkoYc4qtoQPx6uy5qYXR5ZDYVYBSRmCV5piU3JG9)))","sh(wsh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232)))", SIGNABLE, {{"a9147fc63e13dc25e8a95a3cee3d9a714ac3afd96f1e87"}}, OutputType::P2SH_SEGWIT); Check("tr(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,pk(KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy))", "tr(a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,pk(669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0))", "tr(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,pk(KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy))", "tr(a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,pk(669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0))", SIGNABLE | XONLY_KEYS, {{"512017cf18db381d836d8923b1bdb246cfcd818da1a9f0e6e7907f187f0b2f937754"}}, OutputType::BECH32M); CheckUnparsable("sh(multi(16,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGNz6YCCQtYvFzMtrC6D3tKTKdBBboMrLTsjr2NYVBwapCkn7Mr,KxogYhiNfwxuswvXV66eFyKcCpm7dZ7TqHVqujHAVUjJxyivxQ9X,L2BUNduTSyZwZjwNHynQTF14mv2uz2NRq5n5sYWTb4FkkmqgEE9f,L1okJGHGn1kFjdXHKxXjwVVtmCMR2JA5QsbKCSpSb7ReQjezKeoD,KxDCNSST75HFPaW5QKpzHtAyaCQC7p9Vo3FYfi2u4dXD1vgMiboK,L5edQjFtnkcf5UWURn6UuuoFrabgDQUHdheKCziwN42aLwS3KizU,KzF8UWFcEC7BYTq8Go1xVimMkDmyNYVmXV5PV7RuDicvAocoPB8i,L3nHUboKG2w4VSJ5jYZ5CBM97oeK6YuKvfZxrefdShECcjEYKMWZ,KyjHo36dWkYhimKmVVmQTq3gERv3pnqA4xFCpvUgbGDJad7eS8WE,KwsfyHKRUTZPQtysN7M3tZ4GXTnuov5XRgjdF2XCG8faAPmFruRF,KzCUbGhN9LJhdeFfL9zQgTJMjqxdBKEekRGZX24hXdgCNCijkkap,KzgpMBwwsDLwkaC5UrmBgCYaBD2WgZ7PBoGYXR8KT7gCA9UTN5a3,KyBXTPy4T7YG4q9tcAM3LkvfRpD1ybHMvcJ2ehaWXaSqeGUxEdkP,KzJDe9iwJRPtKP2F2AoN6zBgzS7uiuAwhWCfGdNeYJ3PC1HNJ8M8,L1xbHrxynrqLKkoYc4qtoQPx6uy5qYXR5ZDYVYBSRmCV5piU3JG9))","sh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232))", "P2SH script is too large, 547 bytes is larger than 520 bytes"); // P2SH does not fit 16 compressed pubkeys in a redeemscript - CheckUnparsable("wsh(multi(2,[aaaaaaaa][aaaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaa][aaaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multiple ']' characters found for a single pubkey"); // Double key origin descriptor - CheckUnparsable("wsh(multi(2,[aaaagaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaagaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Fingerprint 'aaagaaaa' is not hex"); // Non hex fingerprint - CheckUnparsable("wsh(multi(2,[aaaaaaaa],xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaa],xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "No key provided"); // No public key with origin - CheckUnparsable("wsh(multi(2,[aaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Fingerprint is not 4 bytes (7 characters instead of 8 characters)"); // Too short fingerprint - CheckUnparsable("wsh(multi(2,[aaaaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Fingerprint is not 4 bytes (9 characters instead of 8 characters)"); // Too long fingerprint + CheckUnparsable("wsh(multi(2,[aaaaaaaa][aaaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaa][aaaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multi: Multiple ']' characters found for a single pubkey"); // Double key origin descriptor + CheckUnparsable("wsh(multi(2,[aaaagaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaagaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multi: Fingerprint 'aaagaaaa' is not hex"); // Non hex fingerprint + CheckUnparsable("wsh(multi(2,[aaaaaaaa],xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaa],xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multi: No key provided"); // No public key with origin + CheckUnparsable("wsh(multi(2,[aaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multi: Fingerprint is not 4 bytes (7 characters instead of 8 characters)"); // Too short fingerprint + CheckUnparsable("wsh(multi(2,[aaaaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multi: Fingerprint is not 4 bytes (9 characters instead of 8 characters)"); // Too long fingerprint CheckUnparsable("multi(a,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "multi(a,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "Multi threshold 'a' is not valid"); // Invalid threshold CheckUnparsable("multi(0,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "multi(0,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "Multisig threshold cannot be 0, must be at least 1"); // Threshold of 0 CheckUnparsable("multi(3,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "multi(3,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "Multisig threshold cannot be larger than the number of keys; threshold is 3 but only 2 keys specified"); // Threshold larger than number of keys diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index a490bbfa1d..59adec075e 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -10,7 +10,9 @@ #include <test/util/setup_common.h> #include <util/check.h> #include <util/sock.h> +#include <util/time.h> +#include <csignal> #include <cstdint> #include <exception> #include <fstream> @@ -59,6 +61,7 @@ void FuzzFrameworkRegisterTarget(std::string_view name, TypeTestOneInput target, Assert(it_ins.second); } +static std::string_view g_fuzz_target; static TypeTestOneInput* g_test_one_input{nullptr}; void initialize() @@ -92,9 +95,12 @@ void initialize() should_abort = true; } Assert(!should_abort); - std::string_view fuzz_target{Assert(std::getenv("FUZZ"))}; - const auto it = FuzzTargets().find(fuzz_target); - Assert(it != FuzzTargets().end()); + g_fuzz_target = Assert(std::getenv("FUZZ")); + const auto it = FuzzTargets().find(g_fuzz_target); + if (it == FuzzTargets().end()) { + std::cerr << "No fuzzer for " << g_fuzz_target << "." << std::endl; + std::exit(EXIT_FAILURE); + } Assert(!g_test_one_input); g_test_one_input = &std::get<0>(it->second); std::get<1>(it->second)(); @@ -112,6 +118,35 @@ static bool read_stdin(std::vector<uint8_t>& data) } #endif +#if defined(PROVIDE_FUZZ_MAIN_FUNCTION) && !defined(__AFL_LOOP) +static bool read_file(fs::path p, std::vector<uint8_t>& data) +{ + uint8_t buffer[1024]; + FILE* f = fsbridge::fopen(p, "rb"); + if (f == nullptr) return false; + do { + const size_t length = fread(buffer, sizeof(uint8_t), sizeof(buffer), f); + if (ferror(f)) return false; + data.insert(data.end(), buffer, buffer + length); + } while (!feof(f)); + fclose(f); + return true; +} +#endif + +#if defined(PROVIDE_FUZZ_MAIN_FUNCTION) && !defined(__AFL_LOOP) +static fs::path g_input_path; +void signal_handler(int signal) +{ + if (signal == SIGABRT) { + std::cerr << "Error processing input " << g_input_path << std::endl; + } else { + std::cerr << "Unexpected signal " << signal << " received\n"; + } + std::_Exit(EXIT_FAILURE); +} +#endif + // This function is used by libFuzzer extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { @@ -151,10 +186,37 @@ int main(int argc, char** argv) } #else std::vector<uint8_t> buffer; - if (!read_stdin(buffer)) { + if (argc <= 1) { + if (!read_stdin(buffer)) { + return 0; + } + test_one_input(buffer); return 0; } - test_one_input(buffer); + std::signal(SIGABRT, signal_handler); + int64_t start_time = GetTimeSeconds(); + int tested = 0; + for (int i = 1; i < argc; ++i) { + fs::path input_path(*(argv + i)); + if (fs::is_directory(input_path)) { + for (fs::directory_iterator it(input_path); it != fs::directory_iterator(); ++it) { + if (!fs::is_regular_file(it->path())) continue; + g_input_path = it->path(); + Assert(read_file(it->path(), buffer)); + test_one_input(buffer); + ++tested; + buffer.clear(); + } + } else { + g_input_path = input_path; + Assert(read_file(input_path, buffer)); + test_one_input(buffer); + ++tested; + buffer.clear(); + } + } + int64_t end_time = GetTimeSeconds(); + std::cout << g_fuzz_target << ": succeeded against " << tested << " files in " << (end_time - start_time) << "s." << std::endl; #endif return 0; } diff --git a/src/test/fuzz/miniscript_decode.cpp b/src/test/fuzz/miniscript_decode.cpp new file mode 100644 index 0000000000..4cc0a1be8f --- /dev/null +++ b/src/test/fuzz/miniscript_decode.cpp @@ -0,0 +1,72 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <core_io.h> +#include <hash.h> +#include <key.h> +#include <script/miniscript.h> +#include <script/script.h> +#include <span.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <util/strencodings.h> + +#include <optional> + +using miniscript::operator""_mst; + + +struct Converter { + typedef CPubKey Key; + + bool ToString(const Key& key, std::string& ret) const { + ret = HexStr(key); + return true; + } + const std::vector<unsigned char> ToPKBytes(const Key& key) const { + return {key.begin(), key.end()}; + } + const std::vector<unsigned char> ToPKHBytes(const Key& key) const { + const auto h = Hash160(key); + return {h.begin(), h.end()}; + } + + template<typename I> + bool FromString(I first, I last, Key& key) const { + const auto bytes = ParseHex(std::string(first, last)); + key.Set(bytes.begin(), bytes.end()); + return key.IsValid(); + } + template<typename I> + bool FromPKBytes(I first, I last, CPubKey& key) const { + key.Set(first, last); + return key.IsValid(); + } + template<typename I> + bool FromPKHBytes(I first, I last, CPubKey& key) const { + assert(last - first == 20); + return false; + } +}; + +const Converter CONVERTER; + +FUZZ_TARGET(miniscript_decode) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const std::optional<CScript> script = ConsumeDeserializable<CScript>(fuzzed_data_provider); + if (!script) return; + + const auto ms = miniscript::FromScript(*script, CONVERTER); + if (!ms) return; + + // We can roundtrip it to its string representation. + std::string ms_str; + assert(ms->ToString(CONVERTER, ms_str)); + assert(*miniscript::FromString(ms_str, CONVERTER) == *ms); + // The Script representation must roundtrip since we parsed it this way the first time. + const CScript ms_script = ms->ToScript(CONVERTER); + assert(ms_script == *script); +} diff --git a/src/test/fuzz/net.cpp b/src/test/fuzz/net.cpp index fb11ea36ce..4981287152 100644 --- a/src/test/fuzz/net.cpp +++ b/src/test/fuzz/net.cpp @@ -52,16 +52,6 @@ FUZZ_TARGET_INIT(net, initialize_net) } }, [&] { - const std::optional<CInv> inv_opt = ConsumeDeserializable<CInv>(fuzzed_data_provider); - if (!inv_opt) { - return; - } - node.AddKnownTx(inv_opt->hash); - }, - [&] { - node.PushTxInventory(ConsumeUInt256(fuzzed_data_provider)); - }, - [&] { const std::optional<CService> service_opt = ConsumeDeserializable<CService>(fuzzed_data_provider); if (!service_opt) { return; diff --git a/src/test/fuzz/node_eviction.cpp b/src/test/fuzz/node_eviction.cpp index 2e90085744..6a363f00f7 100644 --- a/src/test/fuzz/node_eviction.cpp +++ b/src/test/fuzz/node_eviction.cpp @@ -26,7 +26,7 @@ FUZZ_TARGET(node_eviction) /*m_last_block_time=*/std::chrono::seconds{fuzzed_data_provider.ConsumeIntegral<int64_t>()}, /*m_last_tx_time=*/std::chrono::seconds{fuzzed_data_provider.ConsumeIntegral<int64_t>()}, /*fRelevantServices=*/fuzzed_data_provider.ConsumeBool(), - /*fRelayTxes=*/fuzzed_data_provider.ConsumeBool(), + /*m_relay_txs=*/fuzzed_data_provider.ConsumeBool(), /*fBloomFilter=*/fuzzed_data_provider.ConsumeBool(), /*nKeyedNetGroup=*/fuzzed_data_provider.ConsumeIntegral<uint64_t>(), /*prefer_evict=*/fuzzed_data_provider.ConsumeBool(), diff --git a/src/test/fuzz/script_format.cpp b/src/test/fuzz/script_format.cpp index 241bdfe666..9186746bcf 100644 --- a/src/test/fuzz/script_format.cpp +++ b/src/test/fuzz/script_format.cpp @@ -29,7 +29,5 @@ FUZZ_TARGET_INIT(script_format, initialize_script_format) (void)ScriptToAsmStr(script, /*fAttemptSighashDecode=*/fuzzed_data_provider.ConsumeBool()); UniValue o1(UniValue::VOBJ); - ScriptPubKeyToUniv(script, o1, /*include_hex=*/fuzzed_data_provider.ConsumeBool()); - UniValue o3(UniValue::VOBJ); - ScriptToUniv(script, o3); + ScriptToUniv(script, /*out=*/o1, /*include_hex=*/fuzzed_data_provider.ConsumeBool(), /*include_address=*/fuzzed_data_provider.ConsumeBool()); } diff --git a/src/test/fuzz/transaction.cpp b/src/test/fuzz/transaction.cpp index 6dd8a36692..273aa0dc5c 100644 --- a/src/test/fuzz/transaction.cpp +++ b/src/test/fuzz/transaction.cpp @@ -102,6 +102,6 @@ FUZZ_TARGET_INIT(transaction, initialize_transaction) (void)IsWitnessStandard(tx, coins_view_cache); UniValue u(UniValue::VOBJ); - TxToUniv(tx, /*hashBlock=*/uint256::ZERO, u); - TxToUniv(tx, /*hashBlock=*/uint256::ONE, u); + TxToUniv(tx, /*block_hash=*/uint256::ZERO, /*entry=*/u); + TxToUniv(tx, /*block_hash=*/uint256::ONE, /*entry=*/u); } diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index f0c1b0d147..6766fbf2d9 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -225,7 +225,7 @@ void FillNode(FuzzedDataProvider& fuzzed_data_provider, ConnmanTestMsg& connman, const ServiceFlags remote_services = ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS); const NetPermissionFlags permission_flags = ConsumeWeakEnum(fuzzed_data_provider, ALL_NET_PERMISSION_FLAGS); const int32_t version = fuzzed_data_provider.ConsumeIntegralInRange<int32_t>(MIN_PEER_PROTO_VERSION, std::numeric_limits<int32_t>::max()); - const bool filter_txs = fuzzed_data_provider.ConsumeBool(); + const bool relay_txs{fuzzed_data_provider.ConsumeBool()}; const CNetMsgMaker mm{0}; @@ -241,7 +241,7 @@ void FillNode(FuzzedDataProvider& fuzzed_data_provider, ConnmanTestMsg& connman, uint64_t{1}, // dummy nonce std::string{}, // dummy subver int32_t{}, // dummy starting_height - filter_txs), + relay_txs), }; (void)connman.ReceiveMsgFrom(node, msg_version); @@ -255,10 +255,9 @@ void FillNode(FuzzedDataProvider& fuzzed_data_provider, ConnmanTestMsg& connman, assert(node.nVersion == version); assert(node.GetCommonVersion() == std::min(version, PROTOCOL_VERSION)); assert(node.nServices == remote_services); - if (node.m_tx_relay != nullptr) { - LOCK(node.m_tx_relay->cs_filter); - assert(node.m_tx_relay->fRelayTxes == filter_txs); - } + CNodeStateStats statestats; + assert(peerman.GetNodeStateStats(node.GetId(), statestats)); + assert(statestats.m_relay_txs == (relay_txs && !node.IsBlockOnlyConn())); node.m_permissionFlags = permission_flags; if (successfully_connected) { CSerializedNetMsg msg_verack{mm.Make(NetMsgType::VERACK)}; diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index c453dae701..f6c1d1efad 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -30,10 +30,10 @@ using node::CBlockTemplate; namespace miner_tests { struct MinerTestingSetup : public TestingSetup { void TestPackageSelection(const CChainParams& chainparams, const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_node.mempool->cs); - bool TestSequenceLocks(const CTransaction& tx, int flags) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_node.mempool->cs) + bool TestSequenceLocks(const CTransaction& tx) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_node.mempool->cs) { CCoinsViewMemPool view_mempool(&m_node.chainman->ActiveChainstate().CoinsTip(), *m_node.mempool); - return CheckSequenceLocks(m_node.chainman->ActiveChain().Tip(), view_mempool, tx, flags); + return CheckSequenceLocksAtTip(m_node.chainman->ActiveChain().Tip(), view_mempool, tx); } BlockAssembler AssemblerForTest(const CChainParams& params); }; @@ -410,7 +410,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) // non-final txs in mempool SetMockTime(m_node.chainman->ActiveChain().Tip()->GetMedianTimePast()+1); - int flags = LOCKTIME_VERIFY_SEQUENCE|LOCKTIME_MEDIAN_TIME_PAST; + const int flags{LOCKTIME_VERIFY_SEQUENCE | LOCKTIME_MEDIAN_TIME_PAST}; // height map std::vector<int> prevheights; @@ -429,8 +429,8 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.nLockTime = 0; hash = tx.GetHash(); m_node.mempool->addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); - BOOST_CHECK(CheckFinalTx(m_node.chainman->ActiveChain().Tip(), CTransaction(tx), flags)); // Locktime passes - BOOST_CHECK(!TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks fail + BOOST_CHECK(CheckFinalTxAtTip(m_node.chainman->ActiveChain().Tip(), CTransaction{tx})); // Locktime passes + BOOST_CHECK(!TestSequenceLocks(CTransaction{tx})); // Sequence locks fail { CBlockIndex* active_chain_tip = m_node.chainman->ActiveChain().Tip(); @@ -443,8 +443,8 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) prevheights[0] = baseheight + 2; hash = tx.GetHash(); m_node.mempool->addUnchecked(entry.Time(GetTime()).FromTx(tx)); - BOOST_CHECK(CheckFinalTx(m_node.chainman->ActiveChain().Tip(), CTransaction(tx), flags)); // Locktime passes - BOOST_CHECK(!TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks fail + BOOST_CHECK(CheckFinalTxAtTip(m_node.chainman->ActiveChain().Tip(), CTransaction{tx})); // Locktime passes + BOOST_CHECK(!TestSequenceLocks(CTransaction{tx})); // Sequence locks fail for (int i = 0; i < CBlockIndex::nMedianTimeSpan; i++) m_node.chainman->ActiveChain().Tip()->GetAncestor(m_node.chainman->ActiveChain().Tip()->nHeight - i)->nTime += 512; //Trick the MedianTimePast @@ -464,8 +464,8 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.nLockTime = m_node.chainman->ActiveChain().Tip()->nHeight + 1; hash = tx.GetHash(); m_node.mempool->addUnchecked(entry.Time(GetTime()).FromTx(tx)); - BOOST_CHECK(!CheckFinalTx(m_node.chainman->ActiveChain().Tip(), CTransaction(tx), flags)); // Locktime fails - BOOST_CHECK(TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks pass + BOOST_CHECK(!CheckFinalTxAtTip(m_node.chainman->ActiveChain().Tip(), CTransaction{tx})); // Locktime fails + BOOST_CHECK(TestSequenceLocks(CTransaction{tx})); // Sequence locks pass BOOST_CHECK(IsFinalTx(CTransaction(tx), m_node.chainman->ActiveChain().Tip()->nHeight + 2, m_node.chainman->ActiveChain().Tip()->GetMedianTimePast())); // Locktime passes on 2nd block // absolute time locked @@ -475,8 +475,8 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) prevheights[0] = baseheight + 4; hash = tx.GetHash(); m_node.mempool->addUnchecked(entry.Time(GetTime()).FromTx(tx)); - BOOST_CHECK(!CheckFinalTx(m_node.chainman->ActiveChain().Tip(), CTransaction(tx), flags)); // Locktime fails - BOOST_CHECK(TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks pass + BOOST_CHECK(!CheckFinalTxAtTip(m_node.chainman->ActiveChain().Tip(), CTransaction{tx})); // Locktime fails + BOOST_CHECK(TestSequenceLocks(CTransaction{tx})); // Sequence locks pass BOOST_CHECK(IsFinalTx(CTransaction(tx), m_node.chainman->ActiveChain().Tip()->nHeight + 2, m_node.chainman->ActiveChain().Tip()->GetMedianTimePast() + 1)); // Locktime passes 1 second later // mempool-dependent transactions (not added) @@ -484,14 +484,14 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) prevheights[0] = m_node.chainman->ActiveChain().Tip()->nHeight + 1; tx.nLockTime = 0; tx.vin[0].nSequence = 0; - BOOST_CHECK(CheckFinalTx(m_node.chainman->ActiveChain().Tip(), CTransaction(tx), flags)); // Locktime passes - BOOST_CHECK(TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks pass + BOOST_CHECK(CheckFinalTxAtTip(m_node.chainman->ActiveChain().Tip(), CTransaction{tx})); // Locktime passes + BOOST_CHECK(TestSequenceLocks(CTransaction{tx})); // Sequence locks pass tx.vin[0].nSequence = 1; - BOOST_CHECK(!TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks fail + BOOST_CHECK(!TestSequenceLocks(CTransaction{tx})); // Sequence locks fail tx.vin[0].nSequence = CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG; - BOOST_CHECK(TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks pass + BOOST_CHECK(TestSequenceLocks(CTransaction{tx})); // Sequence locks pass tx.vin[0].nSequence = CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG | 1; - BOOST_CHECK(!TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks fail + BOOST_CHECK(!TestSequenceLocks(CTransaction{tx})); // Sequence locks fail BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); diff --git a/src/test/miniscript_tests.cpp b/src/test/miniscript_tests.cpp new file mode 100644 index 0000000000..949d30dfd5 --- /dev/null +++ b/src/test/miniscript_tests.cpp @@ -0,0 +1,284 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + + +#include <string> + +#include <test/util/setup_common.h> +#include <boost/test/unit_test.hpp> + +#include <hash.h> +#include <pubkey.h> +#include <uint256.h> +#include <crypto/ripemd160.h> +#include <crypto/sha256.h> +#include <script/miniscript.h> + +namespace { + +/** TestData groups various kinds of precomputed data necessary in this test. */ +struct TestData { + //! The only public keys used in this test. + std::vector<CPubKey> pubkeys; + //! A map from the public keys to their CKeyIDs (faster than hashing every time). + std::map<CPubKey, CKeyID> pkhashes; + std::map<CKeyID, CPubKey> pkmap; + + // Various precomputed hashes + std::vector<std::vector<unsigned char>> sha256; + std::vector<std::vector<unsigned char>> ripemd160; + std::vector<std::vector<unsigned char>> hash256; + std::vector<std::vector<unsigned char>> hash160; + + TestData() + { + // We generate 255 public keys and 255 hashes of each type. + for (int i = 1; i <= 255; ++i) { + // This 32-byte array functions as both private key data and hash preimage (31 zero bytes plus any nonzero byte). + unsigned char keydata[32] = {0}; + keydata[31] = i; + + // Compute CPubkey and CKeyID + CKey key; + key.Set(keydata, keydata + 32, true); + CPubKey pubkey = key.GetPubKey(); + CKeyID keyid = pubkey.GetID(); + pubkeys.push_back(pubkey); + pkhashes.emplace(pubkey, keyid); + pkmap.emplace(keyid, pubkey); + + // Compute various hashes + std::vector<unsigned char> hash; + hash.resize(32); + CSHA256().Write(keydata, 32).Finalize(hash.data()); + sha256.push_back(hash); + CHash256().Write(keydata).Finalize(hash); + hash256.push_back(hash); + hash.resize(20); + CRIPEMD160().Write(keydata, 32).Finalize(hash.data()); + ripemd160.push_back(hash); + CHash160().Write(keydata).Finalize(hash); + hash160.push_back(hash); + } + } +}; + +//! Global TestData object +std::unique_ptr<const TestData> g_testdata; + +/** A class encapsulating conversion routing for CPubKey. */ +struct KeyConverter { + typedef CPubKey Key; + + //! Convert a public key to bytes. + std::vector<unsigned char> ToPKBytes(const CPubKey& key) const { return {key.begin(), key.end()}; } + + //! Convert a public key to its Hash160 bytes (precomputed). + std::vector<unsigned char> ToPKHBytes(const CPubKey& key) const + { + auto it = g_testdata->pkhashes.find(key); + assert(it != g_testdata->pkhashes.end()); + return {it->second.begin(), it->second.end()}; + } + + //! Parse a public key from a range of hex characters. + template<typename I> + bool FromString(I first, I last, CPubKey& key) const { + auto bytes = ParseHex(std::string(first, last)); + key.Set(bytes.begin(), bytes.end()); + return key.IsValid(); + } + + template<typename I> + bool FromPKBytes(I first, I last, CPubKey& key) const { + key.Set(first, last); + return key.IsValid(); + } + + template<typename I> + bool FromPKHBytes(I first, I last, CPubKey& key) const { + assert(last - first == 20); + CKeyID keyid; + std::copy(first, last, keyid.begin()); + auto it = g_testdata->pkmap.find(keyid); + assert(it != g_testdata->pkmap.end()); + key = it->second; + return true; + } +}; + +//! Singleton instance of KeyConverter. +const KeyConverter CONVERTER{}; + +using miniscript::operator"" _mst; + +enum TestMode : int { + TESTMODE_INVALID = 0, + TESTMODE_VALID = 1, + TESTMODE_NONMAL = 2, + TESTMODE_NEEDSIG = 4, + TESTMODE_TIMELOCKMIX = 8 +}; + +void Test(const std::string& ms, const std::string& hexscript, int mode, int opslimit = -1, int stacklimit = -1) +{ + auto node = miniscript::FromString(ms, CONVERTER); + if (mode == TESTMODE_INVALID) { + BOOST_CHECK_MESSAGE(!node || !node->IsValid(), "Unexpectedly valid: " + ms); + } else { + BOOST_CHECK_MESSAGE(node, "Unparseable: " + ms); + BOOST_CHECK_MESSAGE(node->IsValid(), "Invalid: " + ms); + BOOST_CHECK_MESSAGE(node->IsValidTopLevel(), "Invalid top level: " + ms); + auto computed_script = node->ToScript(CONVERTER); + BOOST_CHECK_MESSAGE(node->ScriptSize() == computed_script.size(), "Script size mismatch: " + ms); + if (hexscript != "?") BOOST_CHECK_MESSAGE(HexStr(computed_script) == hexscript, "Script mismatch: " + ms + " (" + HexStr(computed_script) + " vs " + hexscript + ")"); + BOOST_CHECK_MESSAGE(node->IsNonMalleable() == !!(mode & TESTMODE_NONMAL), "Malleability mismatch: " + ms); + BOOST_CHECK_MESSAGE(node->NeedsSignature() == !!(mode & TESTMODE_NEEDSIG), "Signature necessity mismatch: " + ms); + BOOST_CHECK_MESSAGE((node->GetType() << "k"_mst) == !(mode & TESTMODE_TIMELOCKMIX), "Timelock mix mismatch: " + ms); + auto inferred_miniscript = miniscript::FromScript(computed_script, CONVERTER); + BOOST_CHECK_MESSAGE(inferred_miniscript, "Cannot infer miniscript from script: " + ms); + BOOST_CHECK_MESSAGE(inferred_miniscript->ToScript(CONVERTER) == computed_script, "Roundtrip failure: miniscript->script != miniscript->script->miniscript->script: " + ms); + if (opslimit != -1) BOOST_CHECK_MESSAGE((int)node->GetOps() == opslimit, "Ops limit mismatch: " << ms << " (" << node->GetOps() << " vs " << opslimit << ")"); + if (stacklimit != -1) BOOST_CHECK_MESSAGE((int)node->GetStackSize() == stacklimit, "Stack limit mismatch: " << ms << " (" << node->GetStackSize() << " vs " << stacklimit << ")"); + } +} +} // namespace + +BOOST_FIXTURE_TEST_SUITE(miniscript_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(fixed_tests) +{ + g_testdata.reset(new TestData()); + + // Validity rules + Test("l:older(1)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // older(1): valid + Test("l:older(0)", "?", TESTMODE_INVALID); // older(0): k must be at least 1 + Test("l:older(2147483647)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // older(2147483647): valid + Test("l:older(2147483648)", "?", TESTMODE_INVALID); // older(2147483648): k must be below 2^31 + Test("u:after(1)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // after(1): valid + Test("u:after(0)", "?", TESTMODE_INVALID); // after(0): k must be at least 1 + Test("u:after(2147483647)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // after(2147483647): valid + Test("u:after(2147483648)", "?", TESTMODE_INVALID); // after(2147483648): k must be below 2^31 + Test("andor(0,1,1)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // andor(Bdu,B,B): valid + Test("andor(a:0,1,1)", "?", TESTMODE_INVALID); // andor(Wdu,B,B): X must be B + Test("andor(0,a:1,a:1)", "?", TESTMODE_INVALID); // andor(Bdu,W,W): Y and Z must be B/V/K + Test("andor(1,1,1)", "?", TESTMODE_INVALID); // andor(Bu,B,B): X must be d + Test("andor(n:or_i(0,after(1)),1,1)", "?", TESTMODE_VALID); // andor(Bdu,B,B): valid + Test("andor(or_i(0,after(1)),1,1)", "?", TESTMODE_INVALID); // andor(Bd,B,B): X must be u + Test("c:andor(0,pk_k(03a0434d9e47f3c86235477c7b1ae6ae5d3442d49b1943c2b752a68e2a47e247c7),pk_k(036d2b085e9e382ed10b69fc311a03f8641ccfff21574de0927513a49d9a688a00))", "?", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG); // andor(Bdu,K,K): valid + Test("t:andor(0,v:1,v:1)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // andor(Bdu,V,V): valid + Test("and_v(v:1,1)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // and_v(V,B): valid + Test("t:and_v(v:1,v:1)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // and_v(V,V): valid + Test("c:and_v(v:1,pk_k(036d2b085e9e382ed10b69fc311a03f8641ccfff21574de0927513a49d9a688a00))", "?", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG); // and_v(V,K): valid + Test("and_v(1,1)", "?", TESTMODE_INVALID); // and_v(B,B): X must be V + Test("and_v(pk_k(02352bbf4a4cdd12564f93fa332ce333301d9ad40271f8107181340aef25be59d5),1)", "?", TESTMODE_INVALID); // and_v(K,B): X must be V + Test("and_v(v:1,a:1)", "?", TESTMODE_INVALID); // and_v(K,W): Y must be B/V/K + Test("and_b(1,a:1)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // and_b(B,W): valid + Test("and_b(1,1)", "?", TESTMODE_INVALID); // and_b(B,B): Y must W + Test("and_b(v:1,a:1)", "?", TESTMODE_INVALID); // and_b(V,W): X must be B + Test("and_b(a:1,a:1)", "?", TESTMODE_INVALID); // and_b(W,W): X must be B + Test("and_b(pk_k(025601570cb47f238d2b0286db4a990fa0f3ba28d1a319f5e7cf55c2a2444da7cc),a:1)", "?", TESTMODE_INVALID); // and_b(K,W): X must be B + Test("or_b(0,a:0)", "?", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG); // or_b(Bd,Wd): valid + Test("or_b(1,a:0)", "?", TESTMODE_INVALID); // or_b(B,Wd): X must be d + Test("or_b(0,a:1)", "?", TESTMODE_INVALID); // or_b(Bd,W): Y must be d + Test("or_b(0,0)", "?", TESTMODE_INVALID); // or_b(Bd,Bd): Y must W + Test("or_b(v:0,a:0)", "?", TESTMODE_INVALID); // or_b(V,Wd): X must be B + Test("or_b(a:0,a:0)", "?", TESTMODE_INVALID); // or_b(Wd,Wd): X must be B + Test("or_b(pk_k(025601570cb47f238d2b0286db4a990fa0f3ba28d1a319f5e7cf55c2a2444da7cc),a:0)", "?", TESTMODE_INVALID); // or_b(Kd,Wd): X must be B + Test("t:or_c(0,v:1)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // or_c(Bdu,V): valid + Test("t:or_c(a:0,v:1)", "?", TESTMODE_INVALID); // or_c(Wdu,V): X must be B + Test("t:or_c(1,v:1)", "?", TESTMODE_INVALID); // or_c(Bu,V): X must be d + Test("t:or_c(n:or_i(0,after(1)),v:1)", "?", TESTMODE_VALID); // or_c(Bdu,V): valid + Test("t:or_c(or_i(0,after(1)),v:1)", "?", TESTMODE_INVALID); // or_c(Bd,V): X must be u + Test("t:or_c(0,1)", "?", TESTMODE_INVALID); // or_c(Bdu,B): Y must be V + Test("or_d(0,1)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // or_d(Bdu,B): valid + Test("or_d(a:0,1)", "?", TESTMODE_INVALID); // or_d(Wdu,B): X must be B + Test("or_d(1,1)", "?", TESTMODE_INVALID); // or_d(Bu,B): X must be d + Test("or_d(n:or_i(0,after(1)),1)", "?", TESTMODE_VALID); // or_d(Bdu,B): valid + Test("or_d(or_i(0,after(1)),1)", "?", TESTMODE_INVALID); // or_d(Bd,B): X must be u + Test("or_d(0,v:1)", "?", TESTMODE_INVALID); // or_d(Bdu,V): Y must be B + Test("or_i(1,1)", "?", TESTMODE_VALID); // or_i(B,B): valid + Test("t:or_i(v:1,v:1)", "?", TESTMODE_VALID); // or_i(V,V): valid + Test("c:or_i(pk_k(03a0434d9e47f3c86235477c7b1ae6ae5d3442d49b1943c2b752a68e2a47e247c7),pk_k(036d2b085e9e382ed10b69fc311a03f8641ccfff21574de0927513a49d9a688a00))", "?", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG); // or_i(K,K): valid + Test("or_i(a:1,a:1)", "?", TESTMODE_INVALID); // or_i(W,W): X and Y must be B/V/K + Test("or_b(l:after(100),al:after(1000000000))", "?", TESTMODE_VALID); // or_b(timelock, heighlock) valid + Test("and_b(after(100),a:after(1000000000))", "?", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_TIMELOCKMIX); // and_b(timelock, heighlock) invalid + Test("pk(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65)", "2103d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65ac", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG); // alias to c:pk_k + Test("pkh(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65)", "76a914fcd35ddacad9f2d5be5e464639441c6065e6955d88ac", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG); // alias to c:pk_h + + + // Randomly generated test set that covers the majority of type and node type combinations + Test("lltvln:after(1231488000)", "6300676300676300670400046749b1926869516868", TESTMODE_VALID | TESTMODE_NONMAL, 12, 4); + Test("uuj:and_v(v:multi(2,03d01115d548e7561b15c38f004d734633687cf4419620095bc5b0f47070afe85a,025601570cb47f238d2b0286db4a990fa0f3ba28d1a319f5e7cf55c2a2444da7cc),after(1231488000))", "6363829263522103d01115d548e7561b15c38f004d734633687cf4419620095bc5b0f47070afe85a21025601570cb47f238d2b0286db4a990fa0f3ba28d1a319f5e7cf55c2a2444da7cc52af0400046749b168670068670068", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG, 14, 6); + Test("or_b(un:multi(2,03daed4f2be3a8bf278e70132fb0beb7522f570e144bf615c07e996d443dee8729,024ce119c96e2fa357200b559b2f7dd5a5f02d5290aff74b03f3e471b273211c97),al:older(16))", "63522103daed4f2be3a8bf278e70132fb0beb7522f570e144bf615c07e996d443dee872921024ce119c96e2fa357200b559b2f7dd5a5f02d5290aff74b03f3e471b273211c9752ae926700686b63006760b2686c9b", TESTMODE_VALID, 14, 6); + Test("j:and_v(vdv:after(1567547623),older(2016))", "829263766304e7e06e5db169686902e007b268", TESTMODE_VALID | TESTMODE_NONMAL, 11, 2); + Test("t:and_v(vu:hash256(131772552c01444cd81360818376a040b7c3b2b7b0a53550ee3edde216cec61b),v:sha256(ec4916dd28fc4c10d78e287ca5d9cc51ee1ae73cbfde08c6b37324cbfaac8bc5))", "6382012088aa20131772552c01444cd81360818376a040b7c3b2b7b0a53550ee3edde216cec61b876700686982012088a820ec4916dd28fc4c10d78e287ca5d9cc51ee1ae73cbfde08c6b37324cbfaac8bc58851", TESTMODE_VALID | TESTMODE_NONMAL, 12, 4); + Test("t:andor(multi(3,02d7924d4f7d43ea965a465ae3095ff41131e5946f3c85f79e44adbcf8e27e080e,03fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556,02e493dbf1c10d80f3581e4904930b1404cc6c13900ee0758474fa94abe8c4cd13),v:older(4194305),v:sha256(9267d3dbed802941483f1afa2a6bc68de5f653128aca9bf1461c5d0a3ad36ed2))", "532102d7924d4f7d43ea965a465ae3095ff41131e5946f3c85f79e44adbcf8e27e080e2103fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a14602975562102e493dbf1c10d80f3581e4904930b1404cc6c13900ee0758474fa94abe8c4cd1353ae6482012088a8209267d3dbed802941483f1afa2a6bc68de5f653128aca9bf1461c5d0a3ad36ed2886703010040b2696851", TESTMODE_VALID | TESTMODE_NONMAL, 13, 6); + Test("or_d(multi(1,02f9308a019258c31049344f85f89d5229b531c845836f99b08601f113bce036f9),or_b(multi(3,022f01e5e15cca351daff3843fb70f3c2f0a1bdd05e5af888a67784ef3e10a2a01,032fa2104d6b38d11b0230010559879124e42ab8dfeff5ff29dc9cdadd4ecacc3f,03d01115d548e7561b15c38f004d734633687cf4419620095bc5b0f47070afe85a),su:after(500000)))", "512102f9308a019258c31049344f85f89d5229b531c845836f99b08601f113bce036f951ae73645321022f01e5e15cca351daff3843fb70f3c2f0a1bdd05e5af888a67784ef3e10a2a0121032fa2104d6b38d11b0230010559879124e42ab8dfeff5ff29dc9cdadd4ecacc3f2103d01115d548e7561b15c38f004d734633687cf4419620095bc5b0f47070afe85a53ae7c630320a107b16700689b68", TESTMODE_VALID | TESTMODE_NONMAL, 15, 8); + Test("or_d(sha256(38df1c1f64a24a77b23393bca50dff872e31edc4f3b5aa3b90ad0b82f4f089b6),and_n(un:after(499999999),older(4194305)))", "82012088a82038df1c1f64a24a77b23393bca50dff872e31edc4f3b5aa3b90ad0b82f4f089b68773646304ff64cd1db19267006864006703010040b26868", TESTMODE_VALID, 16, 2); + Test("and_v(or_i(v:multi(2,02c6047f9441ed7d6d3045406e95c07cd85c778e4b8cef3ca7abac09b95c709ee5,03774ae7f858a9411e5ef4246b70c65aac5649980be5c17891bbec17895da008cb),v:multi(2,03e60fce93b59e9ec53011aabc21c23e97b2a31369b87a5ae9c44ee89e2a6dec0a,025cbdf0646e5db4eaa398f365f2ea7a0e3d419b7e0330e39ce92bddedcac4f9bc)),sha256(d1ec675902ef1633427ca360b290b0b3045a0d9058ddb5e648b4c3c3224c5c68))", "63522102c6047f9441ed7d6d3045406e95c07cd85c778e4b8cef3ca7abac09b95c709ee52103774ae7f858a9411e5ef4246b70c65aac5649980be5c17891bbec17895da008cb52af67522103e60fce93b59e9ec53011aabc21c23e97b2a31369b87a5ae9c44ee89e2a6dec0a21025cbdf0646e5db4eaa398f365f2ea7a0e3d419b7e0330e39ce92bddedcac4f9bc52af6882012088a820d1ec675902ef1633427ca360b290b0b3045a0d9058ddb5e648b4c3c3224c5c6887", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG, 11, 6); + Test("j:and_b(multi(2,0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798,024ce119c96e2fa357200b559b2f7dd5a5f02d5290aff74b03f3e471b273211c97),s:or_i(older(1),older(4252898)))", "82926352210279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f8179821024ce119c96e2fa357200b559b2f7dd5a5f02d5290aff74b03f3e471b273211c9752ae7c6351b26703e2e440b2689a68", TESTMODE_VALID | TESTMODE_NEEDSIG, 14, 5); + Test("and_b(older(16),s:or_d(sha256(e38990d0c7fc009880a9c07c23842e886c6bbdc964ce6bdd5817ad357335ee6f),n:after(1567547623)))", "60b27c82012088a820e38990d0c7fc009880a9c07c23842e886c6bbdc964ce6bdd5817ad357335ee6f87736404e7e06e5db192689a", TESTMODE_VALID, 12, 2); + Test("j:and_v(v:hash160(20195b5a3d650c17f0f29f91c33f8f6335193d07),or_d(sha256(96de8fc8c256fa1e1556d41af431cace7dca68707c78dd88c3acab8b17164c47),older(16)))", "82926382012088a91420195b5a3d650c17f0f29f91c33f8f6335193d078882012088a82096de8fc8c256fa1e1556d41af431cace7dca68707c78dd88c3acab8b17164c4787736460b26868", TESTMODE_VALID, 16, 3); + Test("and_b(hash256(32ba476771d01e37807990ead8719f08af494723de1d228f2c2c07cc0aa40bac),a:and_b(hash256(131772552c01444cd81360818376a040b7c3b2b7b0a53550ee3edde216cec61b),a:older(1)))", "82012088aa2032ba476771d01e37807990ead8719f08af494723de1d228f2c2c07cc0aa40bac876b82012088aa20131772552c01444cd81360818376a040b7c3b2b7b0a53550ee3edde216cec61b876b51b26c9a6c9a", TESTMODE_VALID | TESTMODE_NONMAL, 15, 3); + Test("thresh(2,multi(2,03a0434d9e47f3c86235477c7b1ae6ae5d3442d49b1943c2b752a68e2a47e247c7,036d2b085e9e382ed10b69fc311a03f8641ccfff21574de0927513a49d9a688a00),a:multi(1,036d2b085e9e382ed10b69fc311a03f8641ccfff21574de0927513a49d9a688a00),ac:pk_k(022f01e5e15cca351daff3843fb70f3c2f0a1bdd05e5af888a67784ef3e10a2a01))", "522103a0434d9e47f3c86235477c7b1ae6ae5d3442d49b1943c2b752a68e2a47e247c721036d2b085e9e382ed10b69fc311a03f8641ccfff21574de0927513a49d9a688a0052ae6b5121036d2b085e9e382ed10b69fc311a03f8641ccfff21574de0927513a49d9a688a0051ae6c936b21022f01e5e15cca351daff3843fb70f3c2f0a1bdd05e5af888a67784ef3e10a2a01ac6c935287", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG, 13, 7); + Test("and_n(sha256(d1ec675902ef1633427ca360b290b0b3045a0d9058ddb5e648b4c3c3224c5c68),t:or_i(v:older(4252898),v:older(144)))", "82012088a820d1ec675902ef1633427ca360b290b0b3045a0d9058ddb5e648b4c3c3224c5c68876400676303e2e440b26967029000b269685168", TESTMODE_VALID, 14, 3); + Test("or_d(d:and_v(v:older(4252898),v:older(4252898)),sha256(38df1c1f64a24a77b23393bca50dff872e31edc4f3b5aa3b90ad0b82f4f089b6))", "766303e2e440b26903e2e440b26968736482012088a82038df1c1f64a24a77b23393bca50dff872e31edc4f3b5aa3b90ad0b82f4f089b68768", TESTMODE_VALID, 14, 3); + Test("c:and_v(or_c(sha256(9267d3dbed802941483f1afa2a6bc68de5f653128aca9bf1461c5d0a3ad36ed2),v:multi(1,02c44d12c7065d812e8acf28d7cbb19f9011ecd9e9fdf281b0e6a3b5e87d22e7db)),pk_k(03acd484e2f0c7f65309ad178a9f559abde09796974c57e714c35f110dfc27ccbe))", "82012088a8209267d3dbed802941483f1afa2a6bc68de5f653128aca9bf1461c5d0a3ad36ed28764512102c44d12c7065d812e8acf28d7cbb19f9011ecd9e9fdf281b0e6a3b5e87d22e7db51af682103acd484e2f0c7f65309ad178a9f559abde09796974c57e714c35f110dfc27ccbeac", TESTMODE_VALID | TESTMODE_NEEDSIG, 8, 3); + Test("c:and_v(or_c(multi(2,036d2b085e9e382ed10b69fc311a03f8641ccfff21574de0927513a49d9a688a00,02352bbf4a4cdd12564f93fa332ce333301d9ad40271f8107181340aef25be59d5),v:ripemd160(1b0f3c404d12075c68c938f9f60ebea4f74941a0)),pk_k(03fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556))", "5221036d2b085e9e382ed10b69fc311a03f8641ccfff21574de0927513a49d9a688a002102352bbf4a4cdd12564f93fa332ce333301d9ad40271f8107181340aef25be59d552ae6482012088a6141b0f3c404d12075c68c938f9f60ebea4f74941a088682103fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556ac", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG, 10, 6); + Test("and_v(andor(hash256(8a35d9ca92a48eaade6f53a64985e9e2afeb74dcf8acb4c3721e0dc7e4294b25),v:hash256(939894f70e6c3a25da75da0cc2071b4076d9b006563cf635986ada2e93c0d735),v:older(50000)),after(499999999))", "82012088aa208a35d9ca92a48eaade6f53a64985e9e2afeb74dcf8acb4c3721e0dc7e4294b2587640350c300b2696782012088aa20939894f70e6c3a25da75da0cc2071b4076d9b006563cf635986ada2e93c0d735886804ff64cd1db1", TESTMODE_VALID, 14, 3); + Test("andor(hash256(5f8d30e655a7ba0d7596bb3ddfb1d2d20390d23b1845000e1e118b3be1b3f040),j:and_v(v:hash160(3a2bff0da9d96868e66abc4427bea4691cf61ccd),older(4194305)),ripemd160(44d90e2d3714c8663b632fcf0f9d5f22192cc4c8))", "82012088aa205f8d30e655a7ba0d7596bb3ddfb1d2d20390d23b1845000e1e118b3be1b3f040876482012088a61444d90e2d3714c8663b632fcf0f9d5f22192cc4c8876782926382012088a9143a2bff0da9d96868e66abc4427bea4691cf61ccd8803010040b26868", TESTMODE_VALID, 20, 3); + Test("or_i(c:and_v(v:after(500000),pk_k(02c6047f9441ed7d6d3045406e95c07cd85c778e4b8cef3ca7abac09b95c709ee5)),sha256(d9147961436944f43cd99d28b2bbddbf452ef872b30c8279e255e7daafc7f946))", "630320a107b1692102c6047f9441ed7d6d3045406e95c07cd85c778e4b8cef3ca7abac09b95c709ee5ac6782012088a820d9147961436944f43cd99d28b2bbddbf452ef872b30c8279e255e7daafc7f9468768", TESTMODE_VALID | TESTMODE_NONMAL, 10, 3); + Test("thresh(2,c:pk_h(025cbdf0646e5db4eaa398f365f2ea7a0e3d419b7e0330e39ce92bddedcac4f9bc),s:sha256(e38990d0c7fc009880a9c07c23842e886c6bbdc964ce6bdd5817ad357335ee6f),a:hash160(dd69735817e0e3f6f826a9238dc2e291184f0131))", "76a9145dedfbf9ea599dd4e3ca6a80b333c472fd0b3f6988ac7c82012088a820e38990d0c7fc009880a9c07c23842e886c6bbdc964ce6bdd5817ad357335ee6f87936b82012088a914dd69735817e0e3f6f826a9238dc2e291184f0131876c935287", TESTMODE_VALID, 18, 5); + Test("and_n(sha256(9267d3dbed802941483f1afa2a6bc68de5f653128aca9bf1461c5d0a3ad36ed2),uc:and_v(v:older(144),pk_k(03fe72c435413d33d48ac09c9161ba8b09683215439d62b7940502bda8b202e6ce)))", "82012088a8209267d3dbed802941483f1afa2a6bc68de5f653128aca9bf1461c5d0a3ad36ed28764006763029000b2692103fe72c435413d33d48ac09c9161ba8b09683215439d62b7940502bda8b202e6ceac67006868", TESTMODE_VALID | TESTMODE_NEEDSIG, 13, 4); + Test("and_n(c:pk_k(03daed4f2be3a8bf278e70132fb0beb7522f570e144bf615c07e996d443dee8729),and_b(l:older(4252898),a:older(16)))", "2103daed4f2be3a8bf278e70132fb0beb7522f570e144bf615c07e996d443dee8729ac64006763006703e2e440b2686b60b26c9a68", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG | TESTMODE_TIMELOCKMIX, 12, 3); + Test("c:or_i(and_v(v:older(16),pk_h(02d7924d4f7d43ea965a465ae3095ff41131e5946f3c85f79e44adbcf8e27e080e)),pk_h(026a245bf6dc698504c89a20cfded60853152b695336c28063b61c65cbd269e6b4))", "6360b26976a9149fc5dbe5efdce10374a4dd4053c93af540211718886776a9142fbd32c8dd59ee7c17e66cb6ebea7e9846c3040f8868ac", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG, 12, 4); + Test("or_d(c:pk_h(02e493dbf1c10d80f3581e4904930b1404cc6c13900ee0758474fa94abe8c4cd13),andor(c:pk_k(024ce119c96e2fa357200b559b2f7dd5a5f02d5290aff74b03f3e471b273211c97),older(2016),after(1567547623)))", "76a914c42e7ef92fdb603af844d064faad95db9bcdfd3d88ac736421024ce119c96e2fa357200b559b2f7dd5a5f02d5290aff74b03f3e471b273211c97ac6404e7e06e5db16702e007b26868", TESTMODE_VALID | TESTMODE_NONMAL, 13, 4); + Test("c:andor(ripemd160(6ad07d21fd5dfc646f0b30577045ce201616b9ba),pk_h(02d7924d4f7d43ea965a465ae3095ff41131e5946f3c85f79e44adbcf8e27e080e),and_v(v:hash256(8a35d9ca92a48eaade6f53a64985e9e2afeb74dcf8acb4c3721e0dc7e4294b25),pk_h(03d01115d548e7561b15c38f004d734633687cf4419620095bc5b0f47070afe85a)))", "82012088a6146ad07d21fd5dfc646f0b30577045ce201616b9ba876482012088aa208a35d9ca92a48eaade6f53a64985e9e2afeb74dcf8acb4c3721e0dc7e4294b258876a914dd100be7d9aea5721158ebde6d6a1fd8fff93bb1886776a9149fc5dbe5efdce10374a4dd4053c93af5402117188868ac", TESTMODE_VALID | TESTMODE_NEEDSIG, 18, 4); + Test("c:andor(u:ripemd160(6ad07d21fd5dfc646f0b30577045ce201616b9ba),pk_h(03daed4f2be3a8bf278e70132fb0beb7522f570e144bf615c07e996d443dee8729),or_i(pk_h(022f01e5e15cca351daff3843fb70f3c2f0a1bdd05e5af888a67784ef3e10a2a01),pk_h(0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798)))", "6382012088a6146ad07d21fd5dfc646f0b30577045ce201616b9ba87670068646376a9149652d86bedf43ad264362e6e6eba6eb764508127886776a914751e76e8199196d454941c45d1b3a323f1433bd688686776a91420d637c1a6404d2227f3561fdbaff5a680dba6488868ac", TESTMODE_VALID | TESTMODE_NEEDSIG, 23, 5); + Test("c:or_i(andor(c:pk_h(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),pk_h(022f01e5e15cca351daff3843fb70f3c2f0a1bdd05e5af888a67784ef3e10a2a01),pk_h(02c6047f9441ed7d6d3045406e95c07cd85c778e4b8cef3ca7abac09b95c709ee5)),pk_k(02d7924d4f7d43ea965a465ae3095ff41131e5946f3c85f79e44adbcf8e27e080e))", "6376a914fcd35ddacad9f2d5be5e464639441c6065e6955d88ac6476a91406afd46bcdfd22ef94ac122aa11f241244a37ecc886776a9149652d86bedf43ad264362e6e6eba6eb7645081278868672102d7924d4f7d43ea965a465ae3095ff41131e5946f3c85f79e44adbcf8e27e080e68ac", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_NEEDSIG, 17, 6); + Test("thresh(1,c:pk_k(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),altv:after(1000000000),altv:after(100))", "2103d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65ac6b6300670400ca9a3bb16951686c936b6300670164b16951686c935187", TESTMODE_VALID, 18, 4); + Test("thresh(2,c:pk_k(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),ac:pk_k(03fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556),altv:after(1000000000),altv:after(100))", "2103d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65ac6b2103fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556ac6c936b6300670400ca9a3bb16951686c936b6300670164b16951686c935287", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_TIMELOCKMIX, 22, 5); + + // Misc unit tests + // A Script with a non minimal push is invalid + std::vector<unsigned char> nonminpush = ParseHex("0000210232780000feff00ffffffffffff21ff005f00ae21ae00000000060602060406564c2102320000060900fe00005f00ae21ae00100000060606060606000000000000000000000000000000000000000000000000000000000000000000"); + const CScript nonminpush_script(nonminpush.begin(), nonminpush.end()); + BOOST_CHECK(miniscript::FromScript(nonminpush_script, CONVERTER) == nullptr); + // A non-minimal VERIFY (<key> CHECKSIG VERIFY 1) + std::vector<unsigned char> nonminverify = ParseHex("2103a0434d9e47f3c86235477c7b1ae6ae5d3442d49b1943c2b752a68e2a47e247c7ac6951"); + const CScript nonminverify_script(nonminverify.begin(), nonminverify.end()); + BOOST_CHECK(miniscript::FromScript(nonminverify_script, CONVERTER) == nullptr); + // A threshold as large as the number of subs is valid. + Test("thresh(2,c:pk_k(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),altv:after(100))", "2103d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65ac6b6300670164b16951686c935287", TESTMODE_VALID | TESTMODE_NEEDSIG | TESTMODE_NONMAL); + // A threshold of 1 is valid. + Test("thresh(1,c:pk_k(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),sc:pk_k(03fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556))", "2103d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65ac7c2103fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556ac935187", TESTMODE_VALID | TESTMODE_NEEDSIG | TESTMODE_NONMAL); + // A threshold with a k larger than the number of subs is invalid + Test("thresh(3,c:pk_k(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),sc:pk_k(03fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556))", "2103d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65ac7c2103fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556ac935187", TESTMODE_INVALID); + // A threshold with a k null is invalid + Test("thresh(0,c:pk_k(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),sc:pk_k(03fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556))", "2103d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65ac7c2103fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556ac935187", TESTMODE_INVALID); + // For CHECKMULTISIG the OP cost is the number of keys, but the stack size is the number of sigs (+1) + const auto ms_multi = miniscript::FromString("multi(1,03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65,03fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556,0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798)", CONVERTER); + BOOST_CHECK(ms_multi); + BOOST_CHECK_EQUAL(ms_multi->GetOps(), 4); // 3 pubkeys + CMS + BOOST_CHECK_EQUAL(ms_multi->GetStackSize(), 3); // 1 sig + dummy elem + script push + + // Timelock tests + Test("after(100)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // only heightlock + Test("after(1000000000)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // only timelock + Test("or_b(l:after(100),al:after(1000000000))", "?", TESTMODE_VALID); // or_b(timelock, heighlock) valid + Test("and_b(after(100),a:after(1000000000))", "?", TESTMODE_VALID | TESTMODE_NONMAL | TESTMODE_TIMELOCKMIX); // and_b(timelock, heighlock) invalid + /* This is correctly detected as non-malleable but for the wrong reason. The type system assumes that branches 1 and 2 + can be spent together to create a non-malleble witness, but because of mixing of timelocks they cannot be spent together. + But since exactly one of the two after's can be satisfied, the witness involving the key cannot be malleated. + */ + Test("thresh(2,ltv:after(1000000000),altv:after(100),a:pk(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65))", "?", TESTMODE_VALID | TESTMODE_TIMELOCKMIX | TESTMODE_NONMAL); // thresh with k = 2 + // This is actually non-malleable in practice, but we cannot detect it in type system. See above rationale + Test("thresh(1,c:pk_k(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),altv:after(1000000000),altv:after(100))", "?", TESTMODE_VALID); // thresh with k = 1 + + + g_testdata.reset(); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/net_peer_eviction_tests.cpp b/src/test/net_peer_eviction_tests.cpp index 6ec3fb0c6b..e5ce936519 100644 --- a/src/test/net_peer_eviction_tests.cpp +++ b/src/test/net_peer_eviction_tests.cpp @@ -627,7 +627,7 @@ BOOST_AUTO_TEST_CASE(peer_eviction_test) number_of_nodes, [number_of_nodes](NodeEvictionCandidate& candidate) { candidate.m_last_block_time = std::chrono::seconds{number_of_nodes - candidate.id}; if (candidate.id <= 7) { - candidate.fRelayTxes = false; + candidate.m_relay_txs = false; candidate.fRelevantServices = true; } }, @@ -646,7 +646,7 @@ BOOST_AUTO_TEST_CASE(peer_eviction_test) number_of_nodes, [number_of_nodes](NodeEvictionCandidate& candidate) { candidate.m_last_block_time = std::chrono::seconds{number_of_nodes - candidate.id}; if (candidate.id <= 7) { - candidate.fRelayTxes = false; + candidate.m_relay_txs = false; candidate.fRelevantServices = true; } }, diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index fcb1a80765..e7c01bd6d0 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -732,26 +732,31 @@ BOOST_AUTO_TEST_CASE(LimitedAndReachable_Network) BOOST_CHECK(IsReachable(NET_IPV6)); BOOST_CHECK(IsReachable(NET_ONION)); BOOST_CHECK(IsReachable(NET_I2P)); + BOOST_CHECK(IsReachable(NET_CJDNS)); SetReachable(NET_IPV4, false); SetReachable(NET_IPV6, false); SetReachable(NET_ONION, false); SetReachable(NET_I2P, false); + SetReachable(NET_CJDNS, false); BOOST_CHECK(!IsReachable(NET_IPV4)); BOOST_CHECK(!IsReachable(NET_IPV6)); BOOST_CHECK(!IsReachable(NET_ONION)); BOOST_CHECK(!IsReachable(NET_I2P)); + BOOST_CHECK(!IsReachable(NET_CJDNS)); SetReachable(NET_IPV4, true); SetReachable(NET_IPV6, true); SetReachable(NET_ONION, true); SetReachable(NET_I2P, true); + SetReachable(NET_CJDNS, true); BOOST_CHECK(IsReachable(NET_IPV4)); BOOST_CHECK(IsReachable(NET_IPV6)); BOOST_CHECK(IsReachable(NET_ONION)); BOOST_CHECK(IsReachable(NET_I2P)); + BOOST_CHECK(IsReachable(NET_CJDNS)); } BOOST_AUTO_TEST_CASE(LimitedAndReachable_NetworkCaseUnroutableAndInternal) diff --git a/src/test/script_segwit_tests.cpp b/src/test/script_segwit_tests.cpp new file mode 100644 index 0000000000..2bad59805f --- /dev/null +++ b/src/test/script_segwit_tests.cpp @@ -0,0 +1,164 @@ +// Copyright (c) 2012-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <script/script.h> +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(script_segwit_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(IsPayToWitnessScriptHash_Valid) +{ + uint256 dummy; + CScript p2wsh; + p2wsh << OP_0 << ToByteVector(dummy); + BOOST_CHECK(p2wsh.IsPayToWitnessScriptHash()); + + std::vector<unsigned char> bytes = {OP_0, 32}; + bytes.insert(bytes.end(), 32, 0); + BOOST_CHECK(CScript(bytes.begin(), bytes.end()).IsPayToWitnessScriptHash()); +} + +BOOST_AUTO_TEST_CASE(IsPayToWitnessScriptHash_Invalid_NotOp0) +{ + uint256 dummy; + CScript notp2wsh; + notp2wsh << OP_1 << ToByteVector(dummy); + BOOST_CHECK(!notp2wsh.IsPayToWitnessScriptHash()); +} + +BOOST_AUTO_TEST_CASE(IsPayToWitnessScriptHash_Invalid_Size) +{ + uint160 dummy; + CScript notp2wsh; + notp2wsh << OP_0 << ToByteVector(dummy); + BOOST_CHECK(!notp2wsh.IsPayToWitnessScriptHash()); +} + +BOOST_AUTO_TEST_CASE(IsPayToWitnessScriptHash_Invalid_Nop) +{ + uint256 dummy; + CScript notp2wsh; + notp2wsh << OP_0 << OP_NOP << ToByteVector(dummy); + BOOST_CHECK(!notp2wsh.IsPayToWitnessScriptHash()); +} + +BOOST_AUTO_TEST_CASE(IsPayToWitnessScriptHash_Invalid_EmptyScript) +{ + CScript notp2wsh; + BOOST_CHECK(!notp2wsh.IsPayToWitnessScriptHash()); +} + +BOOST_AUTO_TEST_CASE(IsPayToWitnessScriptHash_Invalid_Pushdata) +{ + // A script is not P2WSH if OP_PUSHDATA is used to push the hash. + std::vector<unsigned char> bytes = {OP_0, OP_PUSHDATA1, 32}; + bytes.insert(bytes.end(), 32, 0); + BOOST_CHECK(!CScript(bytes.begin(), bytes.end()).IsPayToWitnessScriptHash()); + + bytes = {OP_0, OP_PUSHDATA2, 32, 0}; + bytes.insert(bytes.end(), 32, 0); + BOOST_CHECK(!CScript(bytes.begin(), bytes.end()).IsPayToWitnessScriptHash()); + + bytes = {OP_0, OP_PUSHDATA4, 32, 0, 0, 0}; + bytes.insert(bytes.end(), 32, 0); + BOOST_CHECK(!CScript(bytes.begin(), bytes.end()).IsPayToWitnessScriptHash()); +} + +namespace { + +bool IsExpectedWitnessProgram(const CScript& script, const int expectedVersion, const std::vector<unsigned char>& expectedProgram) +{ + int actualVersion; + std::vector<unsigned char> actualProgram; + if (!script.IsWitnessProgram(actualVersion, actualProgram)) { + return false; + } + BOOST_CHECK_EQUAL(actualVersion, expectedVersion); + BOOST_CHECK(actualProgram == expectedProgram); + return true; +} + +bool IsNoWitnessProgram(const CScript& script) +{ + int dummyVersion; + std::vector<unsigned char> dummyProgram; + return !script.IsWitnessProgram(dummyVersion, dummyProgram); +} + +} // anonymous namespace + +BOOST_AUTO_TEST_CASE(IsWitnessProgram_Valid) +{ + // Witness programs have a minimum data push of 2 bytes. + std::vector<unsigned char> program = {42, 18}; + CScript wit; + wit << OP_0 << program; + BOOST_CHECK(IsExpectedWitnessProgram(wit, 0, program)); + + wit.clear(); + // Witness programs have a maximum data push of 40 bytes. + program.resize(40); + wit << OP_16 << program; + BOOST_CHECK(IsExpectedWitnessProgram(wit, 16, program)); + + program.resize(32); + std::vector<unsigned char> bytes = {OP_5, static_cast<unsigned char>(program.size())}; + bytes.insert(bytes.end(), program.begin(), program.end()); + BOOST_CHECK(IsExpectedWitnessProgram(CScript(bytes.begin(), bytes.end()), 5, program)); +} + +BOOST_AUTO_TEST_CASE(IsWitnessProgram_Invalid_Version) +{ + std::vector<unsigned char> program(10); + CScript nowit; + nowit << OP_1NEGATE << program; + BOOST_CHECK(IsNoWitnessProgram(nowit)); +} + +BOOST_AUTO_TEST_CASE(IsWitnessProgram_Invalid_Size) +{ + std::vector<unsigned char> program(1); + CScript nowit; + nowit << OP_0 << program; + BOOST_CHECK(IsNoWitnessProgram(nowit)); + + nowit.clear(); + program.resize(41); + nowit << OP_0 << program; + BOOST_CHECK(IsNoWitnessProgram(nowit)); +} + +BOOST_AUTO_TEST_CASE(IsWitnessProgram_Invalid_Nop) +{ + std::vector<unsigned char> program(10); + CScript nowit; + nowit << OP_0 << OP_NOP << program; + BOOST_CHECK(IsNoWitnessProgram(nowit)); +} + +BOOST_AUTO_TEST_CASE(IsWitnessProgram_Invalid_EmptyScript) +{ + CScript nowit; + BOOST_CHECK(IsNoWitnessProgram(nowit)); +} + +BOOST_AUTO_TEST_CASE(IsWitnessProgram_Invalid_Pushdata) +{ + // A script is no witness program if OP_PUSHDATA is used to push the hash. + std::vector<unsigned char> bytes = {OP_0, OP_PUSHDATA1, 32}; + bytes.insert(bytes.end(), 32, 0); + BOOST_CHECK(IsNoWitnessProgram(CScript(bytes.begin(), bytes.end()))); + + bytes = {OP_0, OP_PUSHDATA2, 32, 0}; + bytes.insert(bytes.end(), 32, 0); + BOOST_CHECK(IsNoWitnessProgram(CScript(bytes.begin(), bytes.end()))); + + bytes = {OP_0, OP_PUSHDATA4, 32, 0, 0, 0}; + bytes.insert(bytes.end(), 32, 0); + BOOST_CHECK(IsNoWitnessProgram(CScript(bytes.begin(), bytes.end()))); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/system_tests.cpp b/src/test/system_tests.cpp index 9c6950f11f..3f5353b5a2 100644 --- a/src/test/system_tests.cpp +++ b/src/test/system_tests.cpp @@ -12,7 +12,16 @@ // For details see https://github.com/bitcoin/bitcoin/pull/22348. #define __kernel_entry #endif +#if defined(__GNUC__) +// Boost 1.78 requires the following workaround. +// See: https://github.com/boostorg/process/issues/235 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wnarrowing" +#endif #include <boost/process.hpp> +#if defined(__GNUC__) +#pragma GCC diagnostic pop +#endif #endif // ENABLE_EXTERNAL_SIGNER #include <boost/test/unit_test.hpp> diff --git a/src/test/txpackage_tests.cpp b/src/test/txpackage_tests.cpp index 6013080dc6..55919b7f95 100644 --- a/src/test/txpackage_tests.cpp +++ b/src/test/txpackage_tests.cpp @@ -73,19 +73,19 @@ BOOST_FIXTURE_TEST_CASE(package_validation_tests, TestChain100Setup) CKey parent_key; parent_key.MakeNewKey(true); CScript parent_locking_script = GetScriptForDestination(PKHash(parent_key.GetPubKey())); - auto mtx_parent = CreateValidMempoolTransaction(/*input_transaction=*/ m_coinbase_txns[0], /*input_vout=*/0, - /*input_height=*/ 0, /*input_signing_key=*/coinbaseKey, - /*output_destination=*/ parent_locking_script, - /*output_amount=*/ CAmount(49 * COIN), /*submit=*/false); + auto mtx_parent = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[0], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/parent_locking_script, + /*output_amount=*/CAmount(49 * COIN), /*submit=*/false); CTransactionRef tx_parent = MakeTransactionRef(mtx_parent); CKey child_key; child_key.MakeNewKey(true); CScript child_locking_script = GetScriptForDestination(PKHash(child_key.GetPubKey())); - auto mtx_child = CreateValidMempoolTransaction(/*input_transaction=*/ tx_parent, /*input_vout=*/0, - /*input_height=*/ 101, /*input_signing_key=*/parent_key, + auto mtx_child = CreateValidMempoolTransaction(/*input_transaction=*/tx_parent, /*input_vout=*/0, + /*input_height=*/101, /*input_signing_key=*/parent_key, /*output_destination=*/child_locking_script, - /*output_amount=*/ CAmount(48 * COIN), /*submit=*/false); + /*output_amount=*/CAmount(48 * COIN), /*submit=*/false); CTransactionRef tx_child = MakeTransactionRef(mtx_child); const auto result_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, {tx_parent, tx_child}, /*test_accept=*/true); BOOST_CHECK_MESSAGE(result_parent_child.m_state.IsValid(), @@ -128,11 +128,11 @@ BOOST_FIXTURE_TEST_CASE(noncontextual_package_tests, TestChain100Setup) // Parent and Child Package { auto mtx_parent = CreateValidMempoolTransaction(m_coinbase_txns[0], 0, 0, coinbaseKey, spk, - CAmount(49 * COIN), /* submit */ false); + CAmount(49 * COIN), /*submit=*/false); CTransactionRef tx_parent = MakeTransactionRef(mtx_parent); auto mtx_child = CreateValidMempoolTransaction(tx_parent, 0, 101, placeholder_key, spk2, - CAmount(48 * COIN), /* submit */ false); + CAmount(48 * COIN), /*submit=*/false); CTransactionRef tx_child = MakeTransactionRef(mtx_child); PackageValidationState state; @@ -218,14 +218,14 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) // Unrelated transactions are not allowed in package submission. Package package_unrelated; for (size_t i{0}; i < 10; ++i) { - auto mtx = CreateValidMempoolTransaction(/* input_transaction */ m_coinbase_txns[i + 25], /* vout */ 0, - /* input_height */ 0, /* input_signing_key */ coinbaseKey, - /* output_destination */ parent_locking_script, - /* output_amount */ CAmount(49 * COIN), /* submit */ false); + auto mtx = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[i + 25], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/parent_locking_script, + /*output_amount=*/CAmount(49 * COIN), /*submit=*/false); package_unrelated.emplace_back(MakeTransactionRef(mtx)); } auto result_unrelated_submit = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_unrelated, /* test_accept */ false); + package_unrelated, /*test_accept=*/false); BOOST_CHECK(result_unrelated_submit.m_state.IsInvalid()); BOOST_CHECK_EQUAL(result_unrelated_submit.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(result_unrelated_submit.m_state.GetRejectReason(), "package-not-child-with-parents"); @@ -234,10 +234,10 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) // Parent and Child (and Grandchild) Package Package package_parent_child; Package package_3gen; - auto mtx_parent = CreateValidMempoolTransaction(/* input_transaction */ m_coinbase_txns[0], /* vout */ 0, - /* input_height */ 0, /* input_signing_key */ coinbaseKey, - /* output_destination */ parent_locking_script, - /* output_amount */ CAmount(49 * COIN), /* submit */ false); + auto mtx_parent = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[0], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/parent_locking_script, + /*output_amount=*/CAmount(49 * COIN), /*submit=*/false); CTransactionRef tx_parent = MakeTransactionRef(mtx_parent); package_parent_child.push_back(tx_parent); package_3gen.push_back(tx_parent); @@ -245,10 +245,10 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) CKey child_key; child_key.MakeNewKey(true); CScript child_locking_script = GetScriptForDestination(PKHash(child_key.GetPubKey())); - auto mtx_child = CreateValidMempoolTransaction(/* input_transaction */ tx_parent, /* vout */ 0, - /* input_height */ 101, /* input_signing_key */ parent_key, - /* output_destination */ child_locking_script, - /* output_amount */ CAmount(48 * COIN), /* submit */ false); + auto mtx_child = CreateValidMempoolTransaction(/*input_transaction=*/tx_parent, /*input_vout=*/0, + /*input_height=*/101, /*input_signing_key=*/parent_key, + /*output_destination=*/child_locking_script, + /*output_amount=*/CAmount(48 * COIN), /*submit=*/false); CTransactionRef tx_child = MakeTransactionRef(mtx_child); package_parent_child.push_back(tx_child); package_3gen.push_back(tx_child); @@ -256,17 +256,17 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) CKey grandchild_key; grandchild_key.MakeNewKey(true); CScript grandchild_locking_script = GetScriptForDestination(PKHash(grandchild_key.GetPubKey())); - auto mtx_grandchild = CreateValidMempoolTransaction(/* input_transaction */ tx_child, /* vout */ 0, - /* input_height */ 101, /* input_signing_key */ child_key, - /* output_destination */ grandchild_locking_script, - /* output_amount */ CAmount(47 * COIN), /* submit */ false); + auto mtx_grandchild = CreateValidMempoolTransaction(/*input_transaction=*/tx_child, /*input_vout=*/0, + /*input_height=*/101, /*input_signing_key=*/child_key, + /*output_destination=*/grandchild_locking_script, + /*output_amount=*/CAmount(47 * COIN), /*submit=*/false); CTransactionRef tx_grandchild = MakeTransactionRef(mtx_grandchild); package_3gen.push_back(tx_grandchild); // 3 Generations is not allowed. { auto result_3gen_submit = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_3gen, /* test_accept */ false); + package_3gen, /*test_accept=*/false); BOOST_CHECK(result_3gen_submit.m_state.IsInvalid()); BOOST_CHECK_EQUAL(result_3gen_submit.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(result_3gen_submit.m_state.GetRejectReason(), "package-not-child-with-parents"); @@ -280,7 +280,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) package_missing_parent.push_back(MakeTransactionRef(mtx_child)); { const auto result_missing_parent = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_missing_parent, /* test_accept */ false); + package_missing_parent, /*test_accept=*/false); BOOST_CHECK(result_missing_parent.m_state.IsInvalid()); BOOST_CHECK_EQUAL(result_missing_parent.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(result_missing_parent.m_state.GetRejectReason(), "package-not-child-with-unconfirmed-parents"); @@ -291,7 +291,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) // Submit package with parent + child. { const auto submit_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_parent_child, /* test_accept */ false); + package_parent_child, /*test_accept=*/false); expected_pool_size += 2; BOOST_CHECK_MESSAGE(submit_parent_child.m_state.IsValid(), "Package validation unexpectedly failed: " << submit_parent_child.m_state.GetRejectReason()); @@ -310,7 +310,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) // Already-in-mempool transactions should be detected and de-duplicated. { const auto submit_deduped = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - package_parent_child, /* test_accept */ false); + package_parent_child, /*test_accept=*/false); BOOST_CHECK_MESSAGE(submit_deduped.m_state.IsValid(), "Package validation unexpectedly failed: " << submit_deduped.m_state.GetRejectReason()); auto it_parent_deduped = submit_deduped.m_tx_results.find(tx_parent->GetWitnessHash()); @@ -340,10 +340,10 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // and the mempool entry's wtxid returned. CScript witnessScript = CScript() << OP_DROP << OP_TRUE; CScript scriptPubKey = GetScriptForDestination(WitnessV0ScriptHash(witnessScript)); - auto mtx_parent = CreateValidMempoolTransaction(/*input_transaction=*/ m_coinbase_txns[0], /*vout=*/ 0, - /*input_height=*/ 0, /*input_signing_key=*/ coinbaseKey, - /*output_destination=*/ scriptPubKey, - /*output_amount=*/ CAmount(49 * COIN), /*submit=*/ false); + auto mtx_parent = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[0], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/scriptPubKey, + /*output_amount=*/CAmount(49 * COIN), /*submit=*/false); CTransactionRef ptx_parent = MakeTransactionRef(mtx_parent); // Make two children with the same txid but different witnesses. @@ -384,7 +384,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // same-txid-different-witness. { const auto submit_witness1 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_parent, ptx_child1}, /*test_accept=*/ false); + {ptx_parent, ptx_child1}, /*test_accept=*/false); BOOST_CHECK_MESSAGE(submit_witness1.m_state.IsValid(), "Package validation unexpectedly failed: " << submit_witness1.m_state.GetRejectReason()); auto it_parent1 = submit_witness1.m_tx_results.find(ptx_parent->GetWitnessHash()); @@ -400,7 +400,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_child1->GetHash()))); const auto submit_witness2 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_parent, ptx_child2}, /*test_accept=*/ false); + {ptx_parent, ptx_child2}, /*test_accept=*/false); BOOST_CHECK_MESSAGE(submit_witness2.m_state.IsValid(), "Package validation unexpectedly failed: " << submit_witness2.m_state.GetRejectReason()); auto it_parent2_deduped = submit_witness2.m_tx_results.find(ptx_parent->GetWitnessHash()); @@ -417,7 +417,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // Deduplication should work when wtxid != txid. Submit package with the already-in-mempool // transactions again, which should not fail. const auto submit_segwit_dedup = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_parent, ptx_child1}, /*test_accept=*/ false); + {ptx_parent, ptx_child1}, /*test_accept=*/false); BOOST_CHECK_MESSAGE(submit_segwit_dedup.m_state.IsValid(), "Package validation unexpectedly failed: " << submit_segwit_dedup.m_state.GetRejectReason()); auto it_parent_dup = submit_segwit_dedup.m_tx_results.find(ptx_parent->GetWitnessHash()); @@ -436,16 +436,16 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) CKey grandchild_key; grandchild_key.MakeNewKey(true); CScript grandchild_locking_script = GetScriptForDestination(WitnessV0KeyHash(grandchild_key.GetPubKey())); - auto mtx_grandchild = CreateValidMempoolTransaction(/*input_transaction=*/ ptx_child2, /* vout=*/ 0, - /*input_height=*/ 0, /*input_signing_key=*/ child_key, - /*output_destination=*/ grandchild_locking_script, - /*output_amount=*/ CAmount(47 * COIN), /*submit=*/ false); + auto mtx_grandchild = CreateValidMempoolTransaction(/*input_transaction=*/ptx_child2, /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/child_key, + /*output_destination=*/grandchild_locking_script, + /*output_amount=*/CAmount(47 * COIN), /*submit=*/false); CTransactionRef ptx_grandchild = MakeTransactionRef(mtx_grandchild); // We already submitted child1 above. { const auto submit_spend_ignored = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_child2, ptx_grandchild}, /*test_accept=*/ false); + {ptx_child2, ptx_grandchild}, /*test_accept=*/false); BOOST_CHECK_MESSAGE(submit_spend_ignored.m_state.IsValid(), "Package validation unexpectedly failed: " << submit_spend_ignored.m_state.GetRejectReason()); auto it_child2_ignored = submit_spend_ignored.m_tx_results.find(ptx_child2->GetWitnessHash()); @@ -471,10 +471,10 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) acs_witness.stack.push_back(std::vector<unsigned char>(acs_script.begin(), acs_script.end())); // parent1 will already be in the mempool - auto mtx_parent1 = CreateValidMempoolTransaction(/*input_transaction=*/ m_coinbase_txns[1], /*vout=*/ 0, - /*input_height=*/ 0, /*input_signing_key=*/ coinbaseKey, - /*output_destination=*/ acs_spk, - /*output_amount=*/ CAmount(49 * COIN), /*submit=*/ true); + auto mtx_parent1 = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[1], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/acs_spk, + /*output_amount=*/CAmount(49 * COIN), /*submit=*/true); CTransactionRef ptx_parent1 = MakeTransactionRef(mtx_parent1); package_mixed.push_back(ptx_parent1); @@ -489,10 +489,10 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) parent2_witness2.stack.push_back(std::vector<unsigned char>(grandparent2_script.begin(), grandparent2_script.end())); // Create grandparent2 creating an output with multiple spending paths. Submit to mempool. - auto mtx_grandparent2 = CreateValidMempoolTransaction(/*input_transaction=*/ m_coinbase_txns[2], /* vout=*/ 0, - /*input_height=*/ 0, /*input_signing_key=*/ coinbaseKey, - /*output_destination=*/ grandparent2_spk, - /*output_amount=*/ CAmount(49 * COIN), /*submit=*/ true); + auto mtx_grandparent2 = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[2], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/grandparent2_spk, + /*output_amount=*/CAmount(49 * COIN), /*submit=*/true); CTransactionRef ptx_grandparent2 = MakeTransactionRef(mtx_grandparent2); CMutableTransaction mtx_parent2_v1; @@ -517,10 +517,10 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) package_mixed.push_back(ptx_parent2_v1); // parent3 will be a new transaction - auto mtx_parent3 = CreateValidMempoolTransaction(/*input_transaction=*/ m_coinbase_txns[3], /*vout=*/ 0, - /*input_height=*/ 0, /*input_signing_key=*/ coinbaseKey, - /*output_destination=*/ acs_spk, - /*output_amount=*/ CAmount(49 * COIN), /*submit=*/ false); + auto mtx_parent3 = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[3], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/acs_spk, + /*output_amount=*/CAmount(49 * COIN), /*submit=*/false); CTransactionRef ptx_parent3 = MakeTransactionRef(mtx_parent3); package_mixed.push_back(ptx_parent3); diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp index fe3cf52974..62b770753a 100644 --- a/src/test/util/net.cpp +++ b/src/test/util/net.cpp @@ -52,7 +52,7 @@ std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(int n_candida /*m_last_block_time=*/std::chrono::seconds{random_context.randrange(100)}, /*m_last_tx_time=*/std::chrono::seconds{random_context.randrange(100)}, /*fRelevantServices=*/random_context.randbool(), - /*fRelayTxes=*/random_context.randbool(), + /*m_relay_txs=*/random_context.randbool(), /*fBloomFilter=*/random_context.randbool(), /*nKeyedNetGroup=*/random_context.randrange(100), /*prefer_evict=*/random_context.randbool(), diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 1881573e7a..b5d8411e1d 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -78,6 +78,31 @@ BOOST_AUTO_TEST_CASE(util_datadir) BOOST_CHECK_EQUAL(dd_norm, args.GetDataDirBase()); } +namespace { +class NoCopyOrMove +{ +public: + int i; + explicit NoCopyOrMove(int i) : i{i} { } + + NoCopyOrMove() = delete; + NoCopyOrMove(const NoCopyOrMove&) = delete; + NoCopyOrMove(NoCopyOrMove&&) = delete; + NoCopyOrMove& operator=(const NoCopyOrMove&) = delete; + NoCopyOrMove& operator=(NoCopyOrMove&&) = delete; + + operator bool() const { return i != 0; } + + int get_ip1() { return i + 1; } + bool test() + { + // Check that Assume can be used within a lambda and still call methods + [&]() { Assume(get_ip1()); }(); + return Assume(get_ip1() != 5); + } +}; +} // namespace + BOOST_AUTO_TEST_CASE(util_check) { // Check that Assert can forward @@ -89,6 +114,14 @@ BOOST_AUTO_TEST_CASE(util_check) // Check that Assume can be used as unary expression const bool result{Assume(two == 2)}; Assert(result); + + // Check that Assert doesn't require copy/move + NoCopyOrMove x{9}; + Assert(x).i += 3; + Assert(x).test(); + + // Check nested Asserts + BOOST_CHECK_EQUAL(Assert((Assert(x).test() ? 3 : 0)), 3); } BOOST_AUTO_TEST_CASE(util_criticalsection) diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 7ae384ceb3..a15094e5c8 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -53,6 +53,7 @@ static const float RECONNECT_TIMEOUT_EXP = 1.5; * this is belt-and-suspenders sanity limit to prevent memory exhaustion. */ static const int MAX_LINE_LENGTH = 100000; +static const uint16_t DEFAULT_TOR_SOCKS_PORT = 9050; /****** Low-level TorControlConnection ********/ @@ -338,6 +339,73 @@ TorController::~TorController() } } +void TorController::get_socks_cb(TorControlConnection& _conn, const TorControlReply& reply) +{ + // NOTE: We can only get here if -onion is unset + std::string socks_location; + if (reply.code == 250) { + for (const auto& line : reply.lines) { + if (0 == line.compare(0, 20, "net/listeners/socks=")) { + const std::string port_list_str = line.substr(20); + std::vector<std::string> port_list; + boost::split(port_list, port_list_str, boost::is_any_of(" ")); + for (auto& portstr : port_list) { + if (portstr.empty()) continue; + if ((portstr[0] == '"' || portstr[0] == '\'') && portstr.size() >= 2 && (*portstr.rbegin() == portstr[0])) { + portstr = portstr.substr(1, portstr.size() - 2); + if (portstr.empty()) continue; + } + socks_location = portstr; + if (0 == portstr.compare(0, 10, "127.0.0.1:")) { + // Prefer localhost - ignore other ports + break; + } + } + } + } + if (!socks_location.empty()) { + LogPrint(BCLog::TOR, "tor: Get SOCKS port command yielded %s\n", socks_location); + } else { + LogPrintf("tor: Get SOCKS port command returned nothing\n"); + } + } else if (reply.code == 510) { // 510 Unrecognized command + LogPrintf("tor: Get SOCKS port command failed with unrecognized command (You probably should upgrade Tor)\n"); + } else { + LogPrintf("tor: Get SOCKS port command failed; error code %d\n", reply.code); + } + + CService resolved; + Assume(!resolved.IsValid()); + if (!socks_location.empty()) { + resolved = LookupNumeric(socks_location, DEFAULT_TOR_SOCKS_PORT); + } + if (!resolved.IsValid()) { + // Fallback to old behaviour + resolved = LookupNumeric("127.0.0.1", DEFAULT_TOR_SOCKS_PORT); + } + + Assume(resolved.IsValid()); + LogPrint(BCLog::TOR, "tor: Configuring onion proxy for %s\n", resolved.ToStringIPPort()); + Proxy addrOnion = Proxy(resolved, true); + SetProxy(NET_ONION, addrOnion); + + const auto onlynets = gArgs.GetArgs("-onlynet"); + + const bool onion_allowed_by_onlynet{ + !gArgs.IsArgSet("-onlynet") || + std::any_of(onlynets.begin(), onlynets.end(), [](const auto& n) { + return ParseNetwork(n) == NET_ONION; + })}; + + if (onion_allowed_by_onlynet) { + // If NET_ONION is reachable, then the below is a noop. + // + // If NET_ONION is not reachable, then none of -proxy or -onion was given. + // Since we are here, then -torcontrol and -torpassword were given. + SetReachable(NET_ONION, true); + } +} + void TorController::add_onion_cb(TorControlConnection& _conn, const TorControlReply& reply) { if (reply.code == 250) { @@ -381,25 +449,7 @@ void TorController::auth_cb(TorControlConnection& _conn, const TorControlReply& // Now that we know Tor is running setup the proxy for onion addresses // if -onion isn't set to something else. if (gArgs.GetArg("-onion", "") == "") { - CService resolved(LookupNumeric("127.0.0.1", 9050)); - Proxy addrOnion = Proxy(resolved, true); - SetProxy(NET_ONION, addrOnion); - - const auto onlynets = gArgs.GetArgs("-onlynet"); - - const bool onion_allowed_by_onlynet{ - !gArgs.IsArgSet("-onlynet") || - std::any_of(onlynets.begin(), onlynets.end(), [](const auto& n) { - return ParseNetwork(n) == NET_ONION; - })}; - - if (onion_allowed_by_onlynet) { - // If NET_ONION is reachable, then the below is a noop. - // - // If NET_ONION is not reachable, then none of -proxy or -onion was given. - // Since we are here, then -torcontrol and -torpassword were given. - SetReachable(NET_ONION, true); - } + _conn.Command("GETINFO net/listeners/socks", std::bind(&TorController::get_socks_cb, this, std::placeholders::_1, std::placeholders::_2)); } // Finally - now create the service diff --git a/src/torcontrol.h b/src/torcontrol.h index 4ace3edcb1..81475aee74 100644 --- a/src/torcontrol.h +++ b/src/torcontrol.h @@ -140,6 +140,8 @@ private: std::vector<uint8_t> clientNonce; public: + /** Callback for GETINFO net/listeners/socks result */ + void get_socks_cb(TorControlConnection& conn, const TorControlReply& reply); /** Callback for ADD_ONION result */ void add_onion_cb(TorControlConnection& conn, const TorControlReply& reply); /** Callback for AUTHENTICATE result */ diff --git a/src/txdb.cpp b/src/txdb.cpp index 81f34b17aa..afcd1985f5 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -84,7 +84,7 @@ void CCoinsViewDB::ResizeCache(size_t new_cache_size) // filesystem lock. m_db.reset(); m_db = std::make_unique<CDBWrapper>( - m_ldb_path, new_cache_size, m_is_memory, /*fWipe*/ false, /*obfuscate*/ true); + m_ldb_path, new_cache_size, m_is_memory, /*fWipe=*/false, /*obfuscate=*/true); } } diff --git a/src/txmempool.cpp b/src/txmempool.cpp index fb5652d0a0..a480eb038d 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -54,16 +54,6 @@ struct update_ancestor_state int64_t modifySigOpsCost; }; -struct update_fee_delta -{ - explicit update_fee_delta(int64_t _feeDelta) : feeDelta(_feeDelta) { } - - void operator() (CTxMemPoolEntry &e) { e.UpdateFeeDelta(feeDelta); } - -private: - int64_t feeDelta; -}; - bool TestLockPointValidity(CChain& active_chain, const LockPoints& lp) { AssertLockHeld(cs_main); @@ -99,7 +89,7 @@ CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& tx, CAmount fee, nModFeesWithAncestors{nFee}, nSigOpCostWithAncestors{sigOpCost} {} -void CTxMemPoolEntry::UpdateFeeDelta(int64_t newFeeDelta) +void CTxMemPoolEntry::UpdateFeeDelta(CAmount newFeeDelta) { nModFeesWithDescendants += newFeeDelta - feeDelta; nModFeesWithAncestors += newFeeDelta - feeDelta; @@ -338,7 +328,7 @@ bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, staged_ancestors = it->GetMemPoolParentsConst(); } - return CalculateAncestorsAndCheckLimits(entry.GetTxSize(), /* entry_count */ 1, + return CalculateAncestorsAndCheckLimits(entry.GetTxSize(), /*entry_count=*/1, setAncestors, staged_ancestors, limitAncestorCount, limitAncestorSize, limitDescendantCount, limitDescendantSize, errString); @@ -496,7 +486,7 @@ void CTxMemPool::addUnchecked(const CTxMemPoolEntry &entry, setEntries &setAnces CAmount delta{0}; ApplyDelta(entry.GetTx().GetHash(), delta); if (delta) { - mapTx.modify(newit, update_fee_delta(delta)); + mapTx.modify(newit, [&delta](CTxMemPoolEntry& e) { e.UpdateFeeDelta(delta); }); } // Update cachedInnerUsage to include contained transaction's usage. @@ -931,7 +921,7 @@ void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeD delta += nFeeDelta; txiter it = mapTx.find(hash); if (it != mapTx.end()) { - mapTx.modify(it, update_fee_delta(delta)); + mapTx.modify(it, [&delta](CTxMemPoolEntry& e) { e.UpdateFeeDelta(delta); }); // Now update all ancestors' modified fees with descendants setEntries setAncestors; uint64_t nNoLimit = std::numeric_limits<uint64_t>::max(); diff --git a/src/txmempool.h b/src/txmempool.h index e7e5a3c402..f5d5abc62e 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -101,7 +101,7 @@ private: const unsigned int entryHeight; //!< Chain height when entering the mempool const bool spendsCoinbase; //!< keep track of transactions that spend a coinbase const int64_t sigOpCost; //!< Total sigop cost - int64_t feeDelta{0}; //!< Used for determining the priority of the transaction for mining in a block + CAmount feeDelta{0}; //!< Used for determining the priority of the transaction for mining in a block LockPoints lockPoints; //!< Track the height and time at which tx was final // Information about descendants of this transaction that are in the @@ -131,7 +131,7 @@ public: std::chrono::seconds GetTime() const { return std::chrono::seconds{nTime}; } unsigned int GetHeight() const { return entryHeight; } int64_t GetSigOpCost() const { return sigOpCost; } - int64_t GetModifiedFee() const { return nFee + feeDelta; } + CAmount GetModifiedFee() const { return nFee + feeDelta; } size_t DynamicMemoryUsage() const { return nUsageSize; } const LockPoints& GetLockPoints() const { return lockPoints; } @@ -140,8 +140,8 @@ public: // Adjusts the ancestor state void UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps); // Updates the fee delta used for mining priority score, and the - // modified fees with descendants. - void UpdateFeeDelta(int64_t feeDelta); + // modified fees with descendants/ancestors. + void UpdateFeeDelta(CAmount newFeeDelta); // Update the LockPoints after a reorg void UpdateLockPoints(const LockPoints& lp); diff --git a/src/util/check.cpp b/src/util/check.cpp new file mode 100644 index 0000000000..2a9f885560 --- /dev/null +++ b/src/util/check.cpp @@ -0,0 +1,14 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/check.h> + +#include <tinyformat.h> + +void assertion_fail(const char* file, int line, const char* func, const char* assertion) +{ + auto str = strprintf("%s:%s %s: Assertion `%s' failed.\n", file, line, func, assertion); + fwrite(str.data(), 1, str.size(), stderr); + std::abort(); +} diff --git a/src/util/check.h b/src/util/check.h index a443c13cf2..4ee65c8d34 100644 --- a/src/util/check.h +++ b/src/util/check.h @@ -47,14 +47,26 @@ class NonFatalCheckError : public std::runtime_error #endif /** Helper for Assert() */ -template <typename T> -T get_pure_r_value(T&& val) +void assertion_fail(const char* file, int line, const char* func, const char* assertion); + +/** Helper for Assert()/Assume() */ +template <bool IS_ASSERT, typename T> +T&& inline_assertion_check(T&& val, [[maybe_unused]] const char* file, [[maybe_unused]] int line, [[maybe_unused]] const char* func, [[maybe_unused]] const char* assertion) { + if constexpr (IS_ASSERT +#ifdef ABORT_ON_FAILED_ASSUME + || true +#endif + ) { + if (!val) { + assertion_fail(file, line, func, assertion); + } + } return std::forward<T>(val); } /** Identity function. Abort if the value compares equal to zero */ -#define Assert(val) ([&]() -> decltype(get_pure_r_value(val)) { auto&& check = (val); assert(#val && check); return std::forward<decltype(get_pure_r_value(val))>(check); }()) +#define Assert(val) inline_assertion_check<true>(val, __FILE__, __LINE__, __func__, #val) /** * Assume is the identity function. @@ -66,10 +78,6 @@ T get_pure_r_value(T&& val) * - For non-fatal errors in interactive sessions (e.g. RPC or command line * interfaces), CHECK_NONFATAL() might be more appropriate. */ -#ifdef ABORT_ON_FAILED_ASSUME -#define Assume(val) Assert(val) -#else -#define Assume(val) ([&]() -> decltype(get_pure_r_value(val)) { auto&& check = (val); return std::forward<decltype(get_pure_r_value(val))>(check); }()) -#endif +#define Assume(val) inline_assertion_check<false>(val, __FILE__, __LINE__, __func__, #val) #endif // BITCOIN_UTIL_CHECK_H diff --git a/src/util/syscall_sandbox.cpp b/src/util/syscall_sandbox.cpp index f2a9cf664d..a05efac602 100644 --- a/src/util/syscall_sandbox.cpp +++ b/src/util/syscall_sandbox.cpp @@ -592,6 +592,8 @@ public: allowed_syscalls.insert(__NR_getcwd); // get current working directory allowed_syscalls.insert(__NR_getdents); // get directory entries allowed_syscalls.insert(__NR_getdents64); // get directory entries + allowed_syscalls.insert(__NR_inotify_rm_watch);// remove an existing watch from an inotify instance + allowed_syscalls.insert(__NR_linkat); // create relative to a directory file descriptor allowed_syscalls.insert(__NR_lstat); // get file status allowed_syscalls.insert(__NR_mkdir); // create a directory allowed_syscalls.insert(__NR_newfstatat); // get file status diff --git a/src/util/system.cpp b/src/util/system.cpp index 8e45453d31..a7e66defcd 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -6,7 +6,16 @@ #include <util/system.h> #ifdef ENABLE_EXTERNAL_SIGNER +#if defined(__GNUC__) +// Boost 1.78 requires the following workaround. +// See: https://github.com/boostorg/process/issues/235 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wnarrowing" +#endif #include <boost/process.hpp> +#if defined(__GNUC__) +#pragma GCC diagnostic pop +#endif #endif // ENABLE_EXTERNAL_SIGNER #include <chainparamsbase.h> diff --git a/src/util/threadnames.cpp b/src/util/threadnames.cpp index 764fffabd7..a5a86d2598 100644 --- a/src/util/threadnames.cpp +++ b/src/util/threadnames.cpp @@ -6,7 +6,9 @@ #include <config/bitcoin-config.h> #endif +#include <string> #include <thread> +#include <utility> #if (defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)) #include <pthread.h> @@ -16,7 +18,7 @@ #include <util/threadnames.h> #ifdef HAVE_SYS_PRCTL_H -#include <sys/prctl.h> // For prctl, PR_SET_NAME, PR_GET_NAME +#include <sys/prctl.h> #endif //! Set the thread's name at the process level. Does not affect the diff --git a/src/validation.cpp b/src/validation.cpp index d80e2576d2..6db13f1f70 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -65,21 +65,22 @@ using node::BLOCKFILE_CHUNK_SIZE; using node::BlockManager; using node::BlockMap; +using node::CBlockIndexHeightOnlyComparator; using node::CBlockIndexWorkComparator; using node::CCoinsStats; using node::CoinStatsHashType; +using node::fHavePruned; +using node::fImporting; +using node::fPruneMode; +using node::fReindex; using node::GetUTXOStats; +using node::nPruneTarget; using node::OpenBlockFile; using node::ReadBlockFromDisk; using node::SnapshotMetadata; using node::UNDOFILE_CHUNK_SIZE; using node::UndoReadFromDisk; using node::UnlinkPrunedFiles; -using node::fHavePruned; -using node::fImporting; -using node::fPruneMode; -using node::fReindex; -using node::nPruneTarget; #define MICRO 0.000001 #define MILLI 0.001 @@ -107,24 +108,6 @@ const std::vector<std::string> CHECKLEVEL_DOC { "each level includes the checks of the previous levels", }; -bool CBlockIndexWorkComparator::operator()(const CBlockIndex *pa, const CBlockIndex *pb) const { - // First sort by most total work, ... - if (pa->nChainWork > pb->nChainWork) return false; - if (pa->nChainWork < pb->nChainWork) return true; - - // ... then by earliest time received, ... - if (pa->nSequenceId < pb->nSequenceId) return false; - if (pa->nSequenceId > pb->nSequenceId) return true; - - // Use pointer address as tie breaker (should only happen with blocks - // loaded from disk, as those all have id 0). - if (pa < pb) return false; - if (pa > pb) return true; - - // Identical blocks. - return false; -} - /** * Mutex to guard access to validation specific variables, such as reading * or changing the chainstate. @@ -152,14 +135,14 @@ arith_uint256 nMinimumChainWork; CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); -CBlockIndex* CChainState::FindForkInGlobalIndex(const CBlockLocator& locator) const +const CBlockIndex* CChainState::FindForkInGlobalIndex(const CBlockLocator& locator) const { AssertLockHeld(cs_main); // Find the latest block common to locator and chain - we expect that // locator.vHave is sorted descending by height. for (const uint256& hash : locator.vHave) { - CBlockIndex* pindex{m_blockman.LookupBlockIndex(hash)}; + const CBlockIndex* pindex{m_blockman.LookupBlockIndex(hash)}; if (pindex) { if (m_chain.Contains(pindex)) { return pindex; @@ -178,20 +161,12 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, std::vector<CScriptCheck>* pvChecks = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_main); -bool CheckFinalTx(const CBlockIndex* active_chain_tip, const CTransaction &tx, int flags) +bool CheckFinalTxAtTip(const CBlockIndex* active_chain_tip, const CTransaction& tx) { AssertLockHeld(cs_main); assert(active_chain_tip); // TODO: Make active_chain_tip a reference - // By convention a negative value for flags indicates that the - // current network-enforced consensus rules should be used. In - // a future soft-fork scenario that would mean checking which - // rules would be enforced for the next block and setting the - // appropriate flags. At the present time no soft-forks are - // scheduled, so no flags are set. - flags = std::max(flags, 0); - - // CheckFinalTx() uses active_chain_tip.Height()+1 to evaluate + // CheckFinalTxAtTip() uses active_chain_tip.Height()+1 to evaluate // nLockTime because when IsFinalTx() is called within // AcceptBlock(), the height of the block *being* // evaluated is what is used. Thus if we want to know if a @@ -203,18 +178,15 @@ bool CheckFinalTx(const CBlockIndex* active_chain_tip, const CTransaction &tx, i // less than the median time of the previous block they're contained in. // When the next block is created its previous block will be the current // chain tip, so we use that to calculate the median time passed to - // IsFinalTx() if LOCKTIME_MEDIAN_TIME_PAST is set. - const int64_t nBlockTime = (flags & LOCKTIME_MEDIAN_TIME_PAST) - ? active_chain_tip->GetMedianTimePast() - : GetAdjustedTime(); + // IsFinalTx(). + const int64_t nBlockTime{active_chain_tip->GetMedianTimePast()}; return IsFinalTx(tx, nBlockHeight, nBlockTime); } -bool CheckSequenceLocks(CBlockIndex* tip, +bool CheckSequenceLocksAtTip(CBlockIndex* tip, const CCoinsView& coins_view, const CTransaction& tx, - int flags, LockPoints* lp, bool useExistingLockPoints) { @@ -222,7 +194,7 @@ bool CheckSequenceLocks(CBlockIndex* tip, CBlockIndex index; index.pprev = tip; - // CheckSequenceLocks() uses active_chainstate.m_chain.Height()+1 to evaluate + // CheckSequenceLocksAtTip() uses active_chainstate.m_chain.Height()+1 to evaluate // height based locks because when SequenceLocks() is called within // ConnectBlock(), the height of the block *being* // evaluated is what is used. @@ -252,7 +224,7 @@ bool CheckSequenceLocks(CBlockIndex* tip, prevheights[txinIndex] = coin.nHeight; } } - lockPair = CalculateSequenceLocks(tx, flags, prevheights, index); + lockPair = CalculateSequenceLocks(tx, STANDARD_LOCKTIME_VERIFY_FLAGS, prevheights, index); if (lp) { lp->height = lockPair.first; lp->time = lockPair.second; @@ -268,7 +240,7 @@ bool CheckSequenceLocks(CBlockIndex* tip, // lockPair from CalculateSequenceLocks against tip+1. We know // EvaluateSequenceLocks will fail if there was a non-zero sequence // lock on a mempool input, so we can use the return value of - // CheckSequenceLocks to indicate the LockPoints validity + // CheckSequenceLocksAtTip to indicate the LockPoints validity int maxInputHeight = 0; for (const int height : prevheights) { // Can ignore mempool inputs since we'll fail if they had non-zero locks @@ -283,7 +255,7 @@ bool CheckSequenceLocks(CBlockIndex* tip, } // Returns the script flags which should be checked for a given block -static unsigned int GetBlockScriptFlags(const CBlockIndex* pindex, const Consensus::Params& chainparams); +static unsigned int GetBlockScriptFlags(const CBlockIndex& block_index, const Consensus::Params& chainparams); static void LimitMempoolSize(CTxMemPool& pool, CCoinsViewCache& coins_cache, size_t limit, std::chrono::seconds age) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, pool.cs) @@ -358,26 +330,26 @@ void CChainState::MaybeUpdateMempoolForReorg( // Also updates valid entries' cached LockPoints if needed. // If false, the tx is still valid and its lockpoints are updated. // If true, the tx would be invalid in the next block; remove this entry and all of its descendants. - const auto filter_final_and_mature = [this, flags=STANDARD_LOCKTIME_VERIFY_FLAGS](CTxMemPool::txiter it) + const auto filter_final_and_mature = [this](CTxMemPool::txiter it) EXCLUSIVE_LOCKS_REQUIRED(m_mempool->cs, ::cs_main) { AssertLockHeld(m_mempool->cs); AssertLockHeld(::cs_main); const CTransaction& tx = it->GetTx(); // The transaction must be final. - if (!CheckFinalTx(m_chain.Tip(), tx, flags)) return true; + if (!CheckFinalTxAtTip(m_chain.Tip(), tx)) return true; LockPoints lp = it->GetLockPoints(); const bool validLP{TestLockPointValidity(m_chain, lp)}; CCoinsViewMemPool view_mempool(&CoinsTip(), *m_mempool); - // CheckSequenceLocks checks if the transaction will be final in the next block to be + // CheckSequenceLocksAtTip checks if the transaction will be final in the next block to be // created on top of the new chain. We use useExistingLockPoints=false so that, instead of // using the information in lp (which might now refer to a block that no longer exists in // the chain), it will update lp to contain LockPoints relevant to the new chain. - if (!CheckSequenceLocks(m_chain.Tip(), view_mempool, tx, flags, &lp, validLP)) { - // If CheckSequenceLocks fails, remove the tx and don't depend on the LockPoints. + if (!CheckSequenceLocksAtTip(m_chain.Tip(), view_mempool, tx, &lp, validLP)) { + // If CheckSequenceLocksAtTip fails, remove the tx and don't depend on the LockPoints. return true; } else if (!validLP) { - // If CheckSequenceLocks succeeded, it also updated the LockPoints. + // If CheckSequenceLocksAtTip succeeded, it also updated the LockPoints. // Now update the mempool entry lockpoints as well. m_mempool->mapTx.modify(it, [&lp](CTxMemPoolEntry& e) { e.UpdateLockPoints(lp); }); } @@ -528,9 +500,26 @@ public: /* m_package_submission */ true, }; } - // No default ctor to avoid exposing details to clients and allowing the possibility of + + private: + // Private ctor to avoid exposing details to clients and allowing the possibility of // mixing up the order of the arguments. Use static functions above instead. - ATMPArgs() = delete; + ATMPArgs(const CChainParams& chainparams, + int64_t accept_time, + bool bypass_limits, + std::vector<COutPoint>& coins_to_uncache, + bool test_accept, + bool allow_bip125_replacement, + bool package_submission) + : m_chainparams{chainparams}, + m_accept_time{accept_time}, + m_bypass_limits{bypass_limits}, + m_coins_to_uncache{coins_to_uncache}, + m_test_accept{test_accept}, + m_allow_bip125_replacement{allow_bip125_replacement}, + m_package_submission{package_submission} + { + } }; // Single transaction acceptance @@ -705,8 +694,9 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // Only accept nLockTime-using transactions that can be mined in the next // block; we don't want our mempool filled up with transactions that can't // be mined yet. - if (!CheckFinalTx(m_active_chainstate.m_chain.Tip(), tx, STANDARD_LOCKTIME_VERIFY_FLAGS)) + if (!CheckFinalTxAtTip(m_active_chainstate.m_chain.Tip(), tx)) { return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "non-final"); + } if (m_pool.exists(GenTxid::Wtxid(tx.GetWitnessHash()))) { // Exact transaction already exists in the mempool. @@ -786,8 +776,9 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // be mined yet. // Pass in m_view which has all of the relevant inputs cached. Note that, since m_view's // backend was removed, it no longer pulls coins from the mempool. - if (!CheckSequenceLocks(m_active_chainstate.m_chain.Tip(), m_view, tx, STANDARD_LOCKTIME_VERIFY_FLAGS, &lp)) + if (!CheckSequenceLocksAtTip(m_active_chainstate.m_chain.Tip(), m_view, tx, &lp)) { return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "non-BIP68-final"); + } // The mempool holds txs for the next block, so pass height+1 to CheckTxInputs if (!Consensus::CheckTxInputs(tx, state, m_view, m_active_chainstate.m_chain.Height() + 1, ws.m_base_fees)) { @@ -1022,7 +1013,7 @@ bool MemPoolAccept::ConsensusScriptChecks(const ATMPArgs& args, Workspace& ws) // There is a similar check in CreateNewBlock() to prevent creating // invalid blocks (using TestBlockValidity), however allowing such // transactions into the mempool can be exploited as a DoS attack. - unsigned int currentBlockScriptVerifyFlags = GetBlockScriptFlags(m_active_chainstate.m_chain.Tip(), chainparams.GetConsensus()); + unsigned int currentBlockScriptVerifyFlags{GetBlockScriptFlags(*m_active_chainstate.m_chain.Tip(), chainparams.GetConsensus())}; if (!CheckInputsFromMempoolAndCache(tx, state, m_view, m_pool, currentBlockScriptVerifyFlags, ws.m_precomputed_txdata, m_active_chainstate.CoinsTip())) { LogPrintf("BUG! PLEASE REPORT THIS! CheckInputScripts failed against latest-block but not STANDARD flags %s, %s\n", hash.ToString(), state.ToString()); @@ -1863,45 +1854,39 @@ public: static ThresholdConditionCache warningcache[VERSIONBITS_NUM_BITS] GUARDED_BY(cs_main); -static unsigned int GetBlockScriptFlags(const CBlockIndex* pindex, const Consensus::Params& consensusparams) +static unsigned int GetBlockScriptFlags(const CBlockIndex& block_index, const Consensus::Params& consensusparams) { - unsigned int flags = SCRIPT_VERIFY_NONE; - // BIP16 didn't become active until Apr 1 2012 (on mainnet, and // retroactively applied to testnet) // However, only one historical block violated the P2SH rules (on both - // mainnet and testnet), so for simplicity, always leave P2SH - // on except for the one violating block. - if (consensusparams.BIP16Exception.IsNull() || // no bip16 exception on this chain - pindex->phashBlock == nullptr || // this is a new candidate block, eg from TestBlockValidity() - *pindex->phashBlock != consensusparams.BIP16Exception) // this block isn't the historical exception - { - // Enforce WITNESS rules whenever P2SH is in effect - flags |= SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS; + // mainnet and testnet). + // Similarly, only one historical block violated the TAPROOT rules on + // mainnet. + // For simplicity, always leave P2SH+WITNESS+TAPROOT on except for the two + // violating blocks. + uint32_t flags{SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_TAPROOT}; + const auto it{consensusparams.script_flag_exceptions.find(*Assert(block_index.phashBlock))}; + if (it != consensusparams.script_flag_exceptions.end()) { + flags = it->second; } // Enforce the DERSIG (BIP66) rule - if (DeploymentActiveAt(*pindex, consensusparams, Consensus::DEPLOYMENT_DERSIG)) { + if (DeploymentActiveAt(block_index, consensusparams, Consensus::DEPLOYMENT_DERSIG)) { flags |= SCRIPT_VERIFY_DERSIG; } // Enforce CHECKLOCKTIMEVERIFY (BIP65) - if (DeploymentActiveAt(*pindex, consensusparams, Consensus::DEPLOYMENT_CLTV)) { + if (DeploymentActiveAt(block_index, consensusparams, Consensus::DEPLOYMENT_CLTV)) { flags |= SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY; } // Enforce CHECKSEQUENCEVERIFY (BIP112) - if (DeploymentActiveAt(*pindex, consensusparams, Consensus::DEPLOYMENT_CSV)) { + if (DeploymentActiveAt(block_index, consensusparams, Consensus::DEPLOYMENT_CSV)) { flags |= SCRIPT_VERIFY_CHECKSEQUENCEVERIFY; } - // Enforce Taproot (BIP340-BIP342) - if (DeploymentActiveAt(*pindex, consensusparams, Consensus::DEPLOYMENT_TAPROOT)) { - flags |= SCRIPT_VERIFY_TAPROOT; - } - // Enforce BIP147 NULLDUMMY (activated simultaneously with segwit) - if (DeploymentActiveAt(*pindex, consensusparams, Consensus::DEPLOYMENT_SEGWIT)) { + if (DeploymentActiveAt(block_index, consensusparams, Consensus::DEPLOYMENT_SEGWIT)) { flags |= SCRIPT_VERIFY_NULLDUMMY; } @@ -1909,7 +1894,6 @@ static unsigned int GetBlockScriptFlags(const CBlockIndex* pindex, const Consens } - static int64_t nTimeCheck = 0; static int64_t nTimeForks = 0; static int64_t nTimeVerify = 0; @@ -2097,7 +2081,7 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, } // Get the script flags for this block - unsigned int flags = GetBlockScriptFlags(pindex, m_params.GetConsensus()); + unsigned int flags{GetBlockScriptFlags(*pindex, m_params.GetConsensus())}; int64_t nTime2 = GetTimeMicros(); nTimeForks += nTime2 - nTime1; LogPrint(BCLog::BENCH, " - Fork checks: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime2 - nTime1), nTimeForks * MICRO, nTimeForks * MILLI / nBlocksTotal); @@ -3373,7 +3357,7 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidatio // Don't accept any forks from the main chain prior to last checkpoint. // GetLastCheckpoint finds the last checkpoint in MapCheckpoints that's in our // BlockIndex(). - CBlockIndex* pcheckpoint = blockman.GetLastCheckpoint(params.Checkpoints()); + const CBlockIndex* pcheckpoint = blockman.GetLastCheckpoint(params.Checkpoints()); if (pcheckpoint && nHeight < pcheckpoint->nHeight) { LogPrintf("ERROR: %s: forked chain older than last checkpoint (height %d)\n", __func__, nHeight); return state.Invalid(BlockValidationResult::BLOCK_CHECKPOINT, "bad-fork-prior-to-checkpoint"); @@ -4083,8 +4067,74 @@ bool ChainstateManager::LoadBlockIndex() // Load block index from databases bool needs_init = fReindex; if (!fReindex) { - bool ret = m_blockman.LoadBlockIndexDB(*this); + bool ret = m_blockman.LoadBlockIndexDB(); if (!ret) return false; + + std::vector<CBlockIndex*> vSortedByHeight{m_blockman.GetAllBlockIndices()}; + std::sort(vSortedByHeight.begin(), vSortedByHeight.end(), + CBlockIndexHeightOnlyComparator()); + + // Find start of assumed-valid region. + int first_assumed_valid_height = std::numeric_limits<int>::max(); + + for (const CBlockIndex* block : vSortedByHeight) { + if (block->IsAssumedValid()) { + auto chainstates = GetAll(); + + // If we encounter an assumed-valid block index entry, ensure that we have + // one chainstate that tolerates assumed-valid entries and another that does + // not (i.e. the background validation chainstate), since assumed-valid + // entries should always be pending validation by a fully-validated chainstate. + auto any_chain = [&](auto fnc) { return std::any_of(chainstates.cbegin(), chainstates.cend(), fnc); }; + assert(any_chain([](auto chainstate) { return chainstate->reliesOnAssumedValid(); })); + assert(any_chain([](auto chainstate) { return !chainstate->reliesOnAssumedValid(); })); + + first_assumed_valid_height = block->nHeight; + break; + } + } + + for (CBlockIndex* pindex : vSortedByHeight) { + if (ShutdownRequested()) return false; + if (pindex->IsAssumedValid() || + (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && + (pindex->HaveTxsDownloaded() || pindex->pprev == nullptr))) { + + // Fill each chainstate's block candidate set. Only add assumed-valid + // blocks to the tip candidate set if the chainstate is allowed to rely on + // assumed-valid blocks. + // + // If all setBlockIndexCandidates contained the assumed-valid blocks, the + // background chainstate's ActivateBestChain() call would add assumed-valid + // blocks to the chain (based on how FindMostWorkChain() works). Obviously + // we don't want this since the purpose of the background validation chain + // is to validate assued-valid blocks. + // + // Note: This is considering all blocks whose height is greater or equal to + // the first assumed-valid block to be assumed-valid blocks, and excluding + // them from the background chainstate's setBlockIndexCandidates set. This + // does mean that some blocks which are not technically assumed-valid + // (later blocks on a fork beginning before the first assumed-valid block) + // might not get added to the background chainstate, but this is ok, + // because they will still be attached to the active chainstate if they + // actually contain more work. + // + // Instead of this height-based approach, an earlier attempt was made at + // detecting "holistically" whether the block index under consideration + // relied on an assumed-valid ancestor, but this proved to be too slow to + // be practical. + for (CChainState* chainstate : GetAll()) { + if (chainstate->reliesOnAssumedValid() || + pindex->nHeight < first_assumed_valid_height) { + chainstate->setBlockIndexCandidates.insert(pindex); + } + } + } + if (pindex->nStatus & BLOCK_FAILED_MASK && (!m_best_invalid || pindex->nChainWork > m_best_invalid->nChainWork)) { + m_best_invalid = pindex; + } + } + needs_init = m_blockman.m_block_index.empty(); } @@ -4186,7 +4236,7 @@ void CChainState::LoadExternalBlockFile(FILE* fileIn, FlatFilePos* dbp) } // process in case the block isn't known yet - CBlockIndex* pindex = m_blockman.LookupBlockIndex(hash); + const CBlockIndex* pindex = m_blockman.LookupBlockIndex(hash); if (!pindex || (pindex->nStatus & BLOCK_HAVE_DATA) == 0) { BlockValidationState state; if (AcceptBlock(pblock, state, nullptr, true, dbp, nullptr)) { diff --git a/src/validation.h b/src/validation.h index cc2247239f..b13e7f3d8b 100644 --- a/src/validation.h +++ b/src/validation.h @@ -273,16 +273,12 @@ PackageMempoolAcceptResult ProcessNewPackage(CChainState& active_chainstate, CTx const Package& txns, bool test_accept) EXCLUSIVE_LOCKS_REQUIRED(cs_main); -/** Transaction validation functions */ +/* Transaction policy functions */ /** * Check if transaction will be final in the next block to be created. - * - * Calls IsFinalTx() with current block height and appropriate block time. - * - * See consensus/consensus.h for flag definitions. */ -bool CheckFinalTx(const CBlockIndex* active_chain_tip, const CTransaction &tx, int flags = -1) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +bool CheckFinalTxAtTip(const CBlockIndex* active_chain_tip, const CTransaction& tx) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** * Check if transaction will be BIP68 final in the next block to be created on top of tip. @@ -299,14 +295,11 @@ bool CheckFinalTx(const CBlockIndex* active_chain_tip, const CTransaction &tx, i * Optionally stores in LockPoints the resulting height and time calculated and the hash * of the block needed for calculation or skips the calculation and uses the LockPoints * passed in for evaluation. - * The LockPoints should not be considered valid if CheckSequenceLocks returns false. - * - * See consensus/consensus.h for flag definitions. + * The LockPoints should not be considered valid if CheckSequenceLocksAtTip returns false. */ -bool CheckSequenceLocks(CBlockIndex* tip, +bool CheckSequenceLocksAtTip(CBlockIndex* tip, const CCoinsView& coins_view, const CTransaction& tx, - int flags, LockPoints* lp = nullptr, bool useExistingLockPoints = false); @@ -693,7 +686,7 @@ public: bool IsInitialBlockDownload() const; /** Find the last common block of this chain and a locator. */ - CBlockIndex* FindForkInGlobalIndex(const CBlockLocator& locator) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); + const CBlockIndex* FindForkInGlobalIndex(const CBlockLocator& locator) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** * Make various assertions about the state of the block index. @@ -836,7 +829,7 @@ private: bool m_snapshot_validated{false}; CBlockIndex* m_best_invalid; - friend bool node::BlockManager::LoadBlockIndex(const Consensus::Params&, ChainstateManager&); + friend bool node::BlockManager::LoadBlockIndex(const Consensus::Params&); //! Internal helper for ActivateSnapshot(). [[nodiscard]] bool PopulateAndValidateSnapshot( diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index 49f0abf9e7..0d0456af4b 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -60,12 +60,12 @@ bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId& rhs) const * erases the weak pointer from the g_dbenvs map. * @post A new BerkeleyEnvironment weak pointer is inserted into g_dbenvs if the directory path key was not already in the map. */ -std::shared_ptr<BerkeleyEnvironment> GetBerkeleyEnv(const fs::path& env_directory) +std::shared_ptr<BerkeleyEnvironment> GetBerkeleyEnv(const fs::path& env_directory, bool use_shared_memory) { LOCK(cs_db); auto inserted = g_dbenvs.emplace(fs::PathToString(env_directory), std::weak_ptr<BerkeleyEnvironment>()); if (inserted.second) { - auto env = std::make_shared<BerkeleyEnvironment>(env_directory); + auto env = std::make_shared<BerkeleyEnvironment>(env_directory, use_shared_memory); inserted.first->second = env; return env; } @@ -113,7 +113,7 @@ void BerkeleyEnvironment::Reset() fMockDb = false; } -BerkeleyEnvironment::BerkeleyEnvironment(const fs::path& dir_path) : strPath(fs::PathToString(dir_path)) +BerkeleyEnvironment::BerkeleyEnvironment(const fs::path& dir_path, bool use_shared_memory) : strPath(fs::PathToString(dir_path)), m_use_shared_memory(use_shared_memory) { Reset(); } @@ -145,8 +145,9 @@ bool BerkeleyEnvironment::Open(bilingual_str& err) LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", fs::PathToString(pathLogDir), fs::PathToString(pathErrorFile)); unsigned int nEnvFlags = 0; - if (gArgs.GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB)) + if (!m_use_shared_memory) { nEnvFlags |= DB_PRIVATE; + } dbenv->set_lg_dir(fs::PathToString(pathLogDir).c_str()); dbenv->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet @@ -188,7 +189,7 @@ bool BerkeleyEnvironment::Open(bilingual_str& err) } //! Construct an in-memory mock Berkeley environment for testing -BerkeleyEnvironment::BerkeleyEnvironment() +BerkeleyEnvironment::BerkeleyEnvironment() : m_use_shared_memory(false) { Reset(); @@ -377,7 +378,7 @@ void BerkeleyBatch::Flush() nMinutes = 1; if (env) { // env is nullptr for dummy databases (i.e. in tests). Don't actually flush if env is nullptr so we don't segfault - env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetIntArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); + env->dbenv->txn_checkpoint(nMinutes ? m_database.m_max_log_mb * 1024 : 0, nMinutes, 0); } } @@ -831,13 +832,13 @@ std::unique_ptr<BerkeleyDatabase> MakeBerkeleyDatabase(const fs::path& path, con { LOCK(cs_db); // Lock env.m_databases until insert in BerkeleyDatabase constructor std::string data_filename = fs::PathToString(data_file.filename()); - std::shared_ptr<BerkeleyEnvironment> env = GetBerkeleyEnv(data_file.parent_path()); + std::shared_ptr<BerkeleyEnvironment> env = GetBerkeleyEnv(data_file.parent_path(), options.use_shared_memory); if (env->m_databases.count(data_filename)) { error = Untranslated(strprintf("Refusing to load database. Data file '%s' is already loaded.", fs::PathToString(env->Directory() / data_filename))); status = DatabaseStatus::FAILED_ALREADY_LOADED; return nullptr; } - db = std::make_unique<BerkeleyDatabase>(std::move(env), std::move(data_filename)); + db = std::make_unique<BerkeleyDatabase>(std::move(env), std::move(data_filename), options); } if (options.verify && !db->Verify(error)) { diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h index b924890d81..fd6c76183e 100644 --- a/src/wallet/bdb.h +++ b/src/wallet/bdb.h @@ -32,8 +32,6 @@ struct bilingual_str; namespace wallet { -static const unsigned int DEFAULT_WALLET_DBLOGSIZE = 100; -static const bool DEFAULT_WALLET_PRIVDB = true; struct WalletDatabaseFileId { u_int8_t value[DB_FILE_ID_LEN]; @@ -56,8 +54,9 @@ public: std::map<std::string, std::reference_wrapper<BerkeleyDatabase>> m_databases; std::unordered_map<std::string, WalletDatabaseFileId> m_fileids; std::condition_variable_any m_db_in_use; + bool m_use_shared_memory; - explicit BerkeleyEnvironment(const fs::path& env_directory); + explicit BerkeleyEnvironment(const fs::path& env_directory, bool use_shared_memory); BerkeleyEnvironment(); ~BerkeleyEnvironment(); void Reset(); @@ -85,7 +84,7 @@ public: }; /** Get BerkeleyEnvironment given a directory path. */ -std::shared_ptr<BerkeleyEnvironment> GetBerkeleyEnv(const fs::path& env_directory); +std::shared_ptr<BerkeleyEnvironment> GetBerkeleyEnv(const fs::path& env_directory, bool use_shared_memory); class BerkeleyBatch; @@ -98,8 +97,8 @@ public: BerkeleyDatabase() = delete; /** Create DB handle to real database */ - BerkeleyDatabase(std::shared_ptr<BerkeleyEnvironment> env, std::string filename) : - WalletDatabase(), env(std::move(env)), strFile(std::move(filename)) + BerkeleyDatabase(std::shared_ptr<BerkeleyEnvironment> env, std::string filename, const DatabaseOptions& options) : + WalletDatabase(), env(std::move(env)), strFile(std::move(filename)), m_max_log_mb(options.max_log_mb) { auto inserted = this->env->m_databases.emplace(strFile, std::ref(*this)); assert(inserted.second); @@ -160,6 +159,7 @@ public: std::unique_ptr<Db> m_db; std::string strFile; + int64_t m_max_log_mb; /** Make a BerkeleyBatch connected to this database */ std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) override; diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 23faad027f..433759e086 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -50,8 +50,8 @@ struct { * The Branch and Bound algorithm is described in detail in Murch's Master Thesis: * https://murch.one/wp-content/uploads/2016/11/erhardt2016coinselection.pdf * - * @param const std::vector<CInputCoin>& utxo_pool The set of UTXOs that we are choosing from. - * These UTXOs will be sorted in descending order by effective value and the CInputCoins' + * @param const std::vector<OutputGroup>& utxo_pool The set of UTXO groups that we are choosing from. + * These UTXO groups will be sorted in descending order by effective value and the OutputGroups' * values are their effective values. * @param const CAmount& selection_target This is the value that we want to select. It is the lower * bound of the range. @@ -66,14 +66,13 @@ std::optional<SelectionResult> SelectCoinsBnB(std::vector<OutputGroup>& utxo_poo { SelectionResult result(selection_target); CAmount curr_value = 0; - - std::vector<bool> curr_selection; // select the utxo at this index - curr_selection.reserve(utxo_pool.size()); + std::vector<size_t> curr_selection; // selected utxo indexes // Calculate curr_available_value CAmount curr_available_value = 0; for (const OutputGroup& utxo : utxo_pool) { - // Assert that this utxo is not negative. It should never be negative, effective value calculation should have removed it + // Assert that this utxo is not negative. It should never be negative, + // effective value calculation should have removed it assert(utxo.GetSelectionAmount() > 0); curr_available_value += utxo.GetSelectionAmount(); } @@ -85,15 +84,15 @@ std::optional<SelectionResult> SelectCoinsBnB(std::vector<OutputGroup>& utxo_poo std::sort(utxo_pool.begin(), utxo_pool.end(), descending); CAmount curr_waste = 0; - std::vector<bool> best_selection; + std::vector<size_t> best_selection; CAmount best_waste = MAX_MONEY; // Depth First search loop for choosing the UTXOs - for (size_t i = 0; i < TOTAL_TRIES; ++i) { + for (size_t curr_try = 0, utxo_pool_index = 0; curr_try < TOTAL_TRIES; ++curr_try, ++utxo_pool_index) { // Conditions for starting a backtrack bool backtrack = false; - if (curr_value + curr_available_value < selection_target || // Cannot possibly reach target with the amount remaining in the curr_available_value. - curr_value > selection_target + cost_of_change || // Selected value is out of range, go back and try other branch + if (curr_value + curr_available_value < selection_target || // Cannot possibly reach target with the amount remaining in the curr_available_value. + curr_value > selection_target + cost_of_change || // Selected value is out of range, go back and try other branch (curr_waste > best_waste && (utxo_pool.at(0).fee - utxo_pool.at(0).long_term_fee) > 0)) { // Don't select things which we know will be more wasteful if the waste is increasing backtrack = true; } else if (curr_value >= selection_target) { // Selected value is within range @@ -104,7 +103,6 @@ std::optional<SelectionResult> SelectCoinsBnB(std::vector<OutputGroup>& utxo_poo // explore any more UTXOs to avoid burning money like that. if (curr_waste <= best_waste) { best_selection = curr_selection; - best_selection.resize(utxo_pool.size()); best_waste = curr_waste; if (best_waste == 0) { break; @@ -114,38 +112,38 @@ std::optional<SelectionResult> SelectCoinsBnB(std::vector<OutputGroup>& utxo_poo backtrack = true; } - // Backtracking, moving backwards - if (backtrack) { - // Walk backwards to find the last included UTXO that still needs to have its omission branch traversed. - while (!curr_selection.empty() && !curr_selection.back()) { - curr_selection.pop_back(); - curr_available_value += utxo_pool.at(curr_selection.size()).GetSelectionAmount(); - } - + if (backtrack) { // Backtracking, moving backwards if (curr_selection.empty()) { // We have walked back to the first utxo and no branch is untraversed. All solutions searched break; } + // Add omitted UTXOs back to lookahead before traversing the omission branch of last included UTXO. + for (--utxo_pool_index; utxo_pool_index > curr_selection.back(); --utxo_pool_index) { + curr_available_value += utxo_pool.at(utxo_pool_index).GetSelectionAmount(); + } + // Output was included on previous iterations, try excluding now. - curr_selection.back() = false; - OutputGroup& utxo = utxo_pool.at(curr_selection.size() - 1); + assert(utxo_pool_index == curr_selection.back()); + OutputGroup& utxo = utxo_pool.at(utxo_pool_index); curr_value -= utxo.GetSelectionAmount(); curr_waste -= utxo.fee - utxo.long_term_fee; + curr_selection.pop_back(); } else { // Moving forwards, continuing down this branch - OutputGroup& utxo = utxo_pool.at(curr_selection.size()); + OutputGroup& utxo = utxo_pool.at(utxo_pool_index); // Remove this utxo from the curr_available_value utxo amount curr_available_value -= utxo.GetSelectionAmount(); - // Avoid searching a branch if the previous UTXO has the same value and same waste and was excluded. Since the ratio of fee to - // long term fee is the same, we only need to check if one of those values match in order to know that the waste is the same. - if (!curr_selection.empty() && !curr_selection.back() && - utxo.GetSelectionAmount() == utxo_pool.at(curr_selection.size() - 1).GetSelectionAmount() && - utxo.fee == utxo_pool.at(curr_selection.size() - 1).fee) { - curr_selection.push_back(false); - } else { + if (curr_selection.empty() || + // The previous index is included and therefore not relevant for exclusion shortcut + (utxo_pool_index - 1) == curr_selection.back() || + // Avoid searching a branch if the previous UTXO has the same value and same waste and was excluded. + // Since the ratio of fee to long term fee is the same, we only need to check if one of those values match in order to know that the waste is the same. + utxo.GetSelectionAmount() != utxo_pool.at(utxo_pool_index - 1).GetSelectionAmount() || + utxo.fee != utxo_pool.at(utxo_pool_index - 1).fee) + { // Inclusion branch first (Largest First Exploration) - curr_selection.push_back(true); + curr_selection.push_back(utxo_pool_index); curr_value += utxo.GetSelectionAmount(); curr_waste += utxo.fee - utxo.long_term_fee; } @@ -158,23 +156,23 @@ std::optional<SelectionResult> SelectCoinsBnB(std::vector<OutputGroup>& utxo_poo } // Set output set - for (size_t i = 0; i < best_selection.size(); ++i) { - if (best_selection.at(i)) { - result.AddInput(utxo_pool.at(i)); - } + for (const size_t& i : best_selection) { + result.AddInput(utxo_pool.at(i)); } + result.ComputeAndSetWaste(CAmount{0}); + assert(best_waste == result.GetWaste()); return result; } -std::optional<SelectionResult> SelectCoinsSRD(const std::vector<OutputGroup>& utxo_pool, CAmount target_value) +std::optional<SelectionResult> SelectCoinsSRD(const std::vector<OutputGroup>& utxo_pool, CAmount target_value, FastRandomContext& rng) { SelectionResult result(target_value); std::vector<size_t> indexes; indexes.resize(utxo_pool.size()); std::iota(indexes.begin(), indexes.end(), 0); - Shuffle(indexes.begin(), indexes.end(), FastRandomContext()); + Shuffle(indexes.begin(), indexes.end(), rng); CAmount selected_eff_value = 0; for (const size_t i : indexes) { @@ -189,16 +187,27 @@ std::optional<SelectionResult> SelectCoinsSRD(const std::vector<OutputGroup>& ut return std::nullopt; } -static void ApproximateBestSubset(const std::vector<OutputGroup>& groups, const CAmount& nTotalLower, const CAmount& nTargetValue, +/** Find a subset of the OutputGroups that is at least as large as, but as close as possible to, the + * target amount; solve subset sum. + * param@[in] groups OutputGroups to choose from, sorted by value in descending order. + * param@[in] nTotalLower Total (effective) value of the UTXOs in groups. + * param@[in] nTargetValue Subset sum target, not including change. + * param@[out] vfBest Boolean vector representing the subset chosen that is closest to + * nTargetValue, with indices corresponding to groups. If the ith + * entry is true, that means the ith group in groups was selected. + * param@[out] nBest Total amount of subset chosen that is closest to nTargetValue. + * param@[in] iterations Maximum number of tries. + */ +static void ApproximateBestSubset(FastRandomContext& insecure_rand, const std::vector<OutputGroup>& groups, + const CAmount& nTotalLower, const CAmount& nTargetValue, std::vector<char>& vfBest, CAmount& nBest, int iterations = 1000) { std::vector<char> vfIncluded; + // Worst case "best" approximation is just all of the groups. vfBest.assign(groups.size(), true); nBest = nTotalLower; - FastRandomContext insecure_rand; - for (int nRep = 0; nRep < iterations && nBest != nTargetValue; nRep++) { vfIncluded.assign(groups.size(), false); @@ -221,6 +230,8 @@ static void ApproximateBestSubset(const std::vector<OutputGroup>& groups, const if (nTotal >= nTargetValue) { fReachedTarget = true; + // If the total is between nTargetValue and nBest, it's our new best + // approximation. if (nTotal < nBest) { nBest = nTotal; @@ -235,22 +246,25 @@ static void ApproximateBestSubset(const std::vector<OutputGroup>& groups, const } } -std::optional<SelectionResult> KnapsackSolver(std::vector<OutputGroup>& groups, const CAmount& nTargetValue) +std::optional<SelectionResult> KnapsackSolver(std::vector<OutputGroup>& groups, const CAmount& nTargetValue, + CAmount change_target, FastRandomContext& rng) { SelectionResult result(nTargetValue); // List of values less than target std::optional<OutputGroup> lowest_larger; + // Groups with selection amount smaller than the target and any change we might produce. + // Don't include groups larger than this, because they will only cause us to overshoot. std::vector<OutputGroup> applicable_groups; CAmount nTotalLower = 0; - Shuffle(groups.begin(), groups.end(), FastRandomContext()); + Shuffle(groups.begin(), groups.end(), rng); for (const OutputGroup& group : groups) { if (group.GetSelectionAmount() == nTargetValue) { result.AddInput(group); return result; - } else if (group.GetSelectionAmount() < nTargetValue + MIN_CHANGE) { + } else if (group.GetSelectionAmount() < nTargetValue + change_target) { applicable_groups.push_back(group); nTotalLower += group.GetSelectionAmount(); } else if (!lowest_larger || group.GetSelectionAmount() < lowest_larger->GetSelectionAmount()) { @@ -276,15 +290,15 @@ std::optional<SelectionResult> KnapsackSolver(std::vector<OutputGroup>& groups, std::vector<char> vfBest; CAmount nBest; - ApproximateBestSubset(applicable_groups, nTotalLower, nTargetValue, vfBest, nBest); - if (nBest != nTargetValue && nTotalLower >= nTargetValue + MIN_CHANGE) { - ApproximateBestSubset(applicable_groups, nTotalLower, nTargetValue + MIN_CHANGE, vfBest, nBest); + ApproximateBestSubset(rng, applicable_groups, nTotalLower, nTargetValue, vfBest, nBest); + if (nBest != nTargetValue && nTotalLower >= nTargetValue + change_target) { + ApproximateBestSubset(rng, applicable_groups, nTotalLower, nTargetValue + change_target, vfBest, nBest); } // If we have a bigger coin and (either the stochastic approximation didn't find a good solution, // or the next bigger coin is closer), return the bigger coin if (lowest_larger && - ((nBest != nTargetValue && nBest < nTargetValue + MIN_CHANGE) || lowest_larger->GetSelectionAmount() <= nBest)) { + ((nBest != nTargetValue && nBest < nTargetValue + change_target) || lowest_larger->GetSelectionAmount() <= nBest)) { result.AddInput(*lowest_larger); } else { for (unsigned int i = 0; i < applicable_groups.size(); i++) { @@ -313,29 +327,29 @@ std::optional<SelectionResult> KnapsackSolver(std::vector<OutputGroup>& groups, ******************************************************************************/ -void OutputGroup::Insert(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants, bool positive_only) { +void OutputGroup::Insert(const COutput& output, size_t ancestors, size_t descendants, bool positive_only) { // Compute the effective value first - const CAmount coin_fee = output.m_input_bytes < 0 ? 0 : m_effective_feerate.GetFee(output.m_input_bytes); + const CAmount coin_fee = output.input_bytes < 0 ? 0 : m_effective_feerate.GetFee(output.input_bytes); const CAmount ev = output.txout.nValue - coin_fee; // Filter for positive only here before adding the coin if (positive_only && ev <= 0) return; m_outputs.push_back(output); - CInputCoin& coin = m_outputs.back(); + COutput& coin = m_outputs.back(); - coin.m_fee = coin_fee; - fee += coin.m_fee; + coin.fee = coin_fee; + fee += coin.fee; - coin.m_long_term_fee = coin.m_input_bytes < 0 ? 0 : m_long_term_feerate.GetFee(coin.m_input_bytes); - long_term_fee += coin.m_long_term_fee; + coin.long_term_fee = coin.input_bytes < 0 ? 0 : m_long_term_feerate.GetFee(coin.input_bytes); + long_term_fee += coin.long_term_fee; coin.effective_value = ev; effective_value += coin.effective_value; - m_from_me &= from_me; - m_value += output.txout.nValue; - m_depth = std::min(m_depth, depth); + m_from_me &= coin.from_me; + m_value += coin.txout.nValue; + m_depth = std::min(m_depth, coin.depth); // ancestors here express the number of ancestors the new coin will end up having, which is // the sum, rather than the max; this will overestimate in the cases where multiple inputs // have common ancestors @@ -357,7 +371,7 @@ CAmount OutputGroup::GetSelectionAmount() const return m_subtract_fee_outputs ? m_value : effective_value; } -CAmount GetSelectionWaste(const std::set<CInputCoin>& inputs, CAmount change_cost, CAmount target, bool use_effective_value) +CAmount GetSelectionWaste(const std::set<COutput>& inputs, CAmount change_cost, CAmount target, bool use_effective_value) { // This function should not be called with empty inputs as that would mean the selection failed assert(!inputs.empty()); @@ -365,8 +379,8 @@ CAmount GetSelectionWaste(const std::set<CInputCoin>& inputs, CAmount change_cos // Always consider the cost of spending an input now vs in the future. CAmount waste = 0; CAmount selected_effective_value = 0; - for (const CInputCoin& coin : inputs) { - waste += coin.m_fee - coin.m_long_term_fee; + for (const COutput& coin : inputs) { + waste += coin.fee - coin.long_term_fee; selected_effective_value += use_effective_value ? coin.effective_value : coin.txout.nValue; } @@ -384,6 +398,17 @@ CAmount GetSelectionWaste(const std::set<CInputCoin>& inputs, CAmount change_cos return waste; } +CAmount GenerateChangeTarget(CAmount payment_value, FastRandomContext& rng) +{ + if (payment_value <= CHANGE_LOWER / 2) { + return CHANGE_LOWER; + } else { + // random value between 50ksat and min (payment_value * 2, 1milsat) + const auto upper_bound = std::min(payment_value * 2, CHANGE_UPPER); + return rng.randrange(upper_bound - CHANGE_LOWER) + CHANGE_LOWER; + } +} + void SelectionResult::ComputeAndSetWaste(CAmount change_cost) { m_waste = GetSelectionWaste(m_selected_inputs, change_cost, m_target, m_use_effective); @@ -411,14 +436,14 @@ void SelectionResult::AddInput(const OutputGroup& group) m_use_effective = !group.m_subtract_fee_outputs; } -const std::set<CInputCoin>& SelectionResult::GetInputSet() const +const std::set<COutput>& SelectionResult::GetInputSet() const { return m_selected_inputs; } -std::vector<CInputCoin> SelectionResult::GetShuffledInputVector() const +std::vector<COutput> SelectionResult::GetShuffledInputVector() const { - std::vector<CInputCoin> coins(m_selected_inputs.begin(), m_selected_inputs.end()); + std::vector<COutput> coins(m_selected_inputs.begin(), m_selected_inputs.end()); Shuffle(coins.begin(), coins.end(), FastRandomContext()); return coins; } @@ -430,4 +455,9 @@ bool SelectionResult::operator<(SelectionResult other) const // As this operator is only used in std::min_element, we want the result that has more inputs when waste are equal. return *m_waste < *other.m_waste || (*m_waste == *other.m_waste && m_selected_inputs.size() > other.m_selected_inputs.size()); } + +std::string COutput::ToString() const +{ + return strprintf("COutput(%s, %d, %d) [%s]", outpoint.hash.ToString(), outpoint.n, depth, FormatMoney(txout.nValue)); +} } // namespace wallet diff --git a/src/wallet/coinselection.h b/src/wallet/coinselection.h index 496a026999..784a91e827 100644 --- a/src/wallet/coinselection.h +++ b/src/wallet/coinselection.h @@ -13,74 +13,93 @@ #include <optional> namespace wallet { -//! target minimum change amount -static constexpr CAmount MIN_CHANGE{COIN / 100}; -//! final minimum change amount after paying for fees -static const CAmount MIN_FINAL_CHANGE = MIN_CHANGE/2; +//! lower bound for randomly-chosen target change amount +static constexpr CAmount CHANGE_LOWER{50000}; +//! upper bound for randomly-chosen target change amount +static constexpr CAmount CHANGE_UPPER{1000000}; /** A UTXO under consideration for use in funding a new transaction. */ -class CInputCoin { -public: - CInputCoin(const CTransactionRef& tx, unsigned int i) - { - if (!tx) - throw std::invalid_argument("tx should not be null"); - if (i >= tx->vout.size()) - throw std::out_of_range("The output index is out of range"); - - outpoint = COutPoint(tx->GetHash(), i); - txout = tx->vout[i]; - effective_value = txout.nValue; - } +struct COutput { + /** The outpoint identifying this UTXO */ + COutPoint outpoint; - CInputCoin(const CTransactionRef& tx, unsigned int i, int input_bytes) : CInputCoin(tx, i) - { - m_input_bytes = input_bytes; - } + /** The output itself */ + CTxOut txout; - CInputCoin(const COutPoint& outpoint_in, const CTxOut& txout_in) - { - outpoint = outpoint_in; - txout = txout_in; - effective_value = txout.nValue; - } + /** + * Depth in block chain. + * If > 0: the tx is on chain and has this many confirmations. + * If = 0: the tx is waiting confirmation. + * If < 0: a conflicting tx is on chain and has this many confirmations. */ + int depth; - CInputCoin(const COutPoint& outpoint_in, const CTxOut& txout_in, int input_bytes) : CInputCoin(outpoint_in, txout_in) - { - m_input_bytes = input_bytes; - } + /** Pre-computed estimated size of this output as a fully-signed input in a transaction. Can be -1 if it could not be calculated */ + int input_bytes; - COutPoint outpoint; - CTxOut txout; + /** Whether we have the private keys to spend this output */ + bool spendable; + + /** Whether we know how to spend this output, ignoring the lack of keys */ + bool solvable; + + /** + * Whether this output is considered safe to spend. Unconfirmed transactions + * from outside keys and unconfirmed replacement transactions are considered + * unsafe and will not be used to fund new spending transactions. + */ + bool safe; + + /** The time of the transaction containing this output as determined by CWalletTx::nTimeSmart */ + int64_t time; + + /** Whether the transaction containing this output is sent from the owning wallet */ + bool from_me; + + /** The output's value minus fees required to spend it. Initialized as the output's absolute value. */ CAmount effective_value; - CAmount m_fee{0}; - CAmount m_long_term_fee{0}; - /** Pre-computed estimated size of this output as a fully-signed input in a transaction. Can be -1 if it could not be calculated */ - int m_input_bytes{-1}; + /** The fee required to spend this output at the transaction's target feerate. */ + CAmount fee{0}; - bool operator<(const CInputCoin& rhs) const { - return outpoint < rhs.outpoint; - } + /** The fee required to spend this output at the consolidation feerate. */ + CAmount long_term_fee{0}; - bool operator!=(const CInputCoin& rhs) const { - return outpoint != rhs.outpoint; - } + COutput(const COutPoint& outpoint, const CTxOut& txout, int depth, int input_bytes, bool spendable, bool solvable, bool safe, int64_t time, bool from_me) + : outpoint{outpoint}, + txout{txout}, + depth{depth}, + input_bytes{input_bytes}, + spendable{spendable}, + solvable{solvable}, + safe{safe}, + time{time}, + from_me{from_me}, + effective_value{txout.nValue} + {} + + std::string ToString() const; - bool operator==(const CInputCoin& rhs) const { - return outpoint == rhs.outpoint; + bool operator<(const COutput& rhs) const + { + return outpoint < rhs.outpoint; } }; /** Parameters for one iteration of Coin Selection. */ -struct CoinSelectionParams -{ +struct CoinSelectionParams { + /** Randomness to use in the context of coin selection. */ + FastRandomContext& rng_fast; /** Size of a change output in bytes, determined by the output type. */ size_t change_output_size = 0; /** Size of the input to spend a change output in virtual bytes. */ size_t change_spend_size = 0; + /** Mininmum change to target in Knapsack solver: select coins to cover the payment and + * at least this value of change. */ + CAmount m_min_change_target{0}; /** Cost of creating the change output. */ CAmount m_change_fee{0}; + /** The pre-determined minimum value to target when funding a change output. */ + CAmount m_change_target{0}; /** Cost of creating the change output + cost of spending the change output in the future. */ CAmount m_cost_of_change{0}; /** The targeted feerate of the transaction being built. */ @@ -100,17 +119,22 @@ struct CoinSelectionParams * reuse. Dust outputs are not eligible to be added to output groups and thus not considered. */ bool m_avoid_partial_spends = false; - CoinSelectionParams(size_t change_output_size, size_t change_spend_size, CFeeRate effective_feerate, - CFeeRate long_term_feerate, CFeeRate discard_feerate, size_t tx_noinputs_size, bool avoid_partial) : - change_output_size(change_output_size), - change_spend_size(change_spend_size), - m_effective_feerate(effective_feerate), - m_long_term_feerate(long_term_feerate), - m_discard_feerate(discard_feerate), - tx_noinputs_size(tx_noinputs_size), - m_avoid_partial_spends(avoid_partial) - {} - CoinSelectionParams() {} + CoinSelectionParams(FastRandomContext& rng_fast, size_t change_output_size, size_t change_spend_size, + CAmount min_change_target, CFeeRate effective_feerate, + CFeeRate long_term_feerate, CFeeRate discard_feerate, size_t tx_noinputs_size, bool avoid_partial) + : rng_fast{rng_fast}, + change_output_size(change_output_size), + change_spend_size(change_spend_size), + m_min_change_target(min_change_target), + m_effective_feerate(effective_feerate), + m_long_term_feerate(long_term_feerate), + m_discard_feerate(discard_feerate), + tx_noinputs_size(tx_noinputs_size), + m_avoid_partial_spends(avoid_partial) + { + } + CoinSelectionParams(FastRandomContext& rng_fast) + : rng_fast{rng_fast} {} }; /** Parameters for filtering which OutputGroups we may use in coin selection. @@ -139,7 +163,7 @@ struct CoinEligibilityFilter struct OutputGroup { /** The list of UTXOs contained in this output group. */ - std::vector<CInputCoin> m_outputs; + std::vector<COutput> m_outputs; /** Whether the UTXOs were sent by the wallet to itself. This is relevant because we may want at * least a certain number of confirmations on UTXOs received from outside wallets while trusting * our own UTXOs more. */ @@ -176,7 +200,7 @@ struct OutputGroup m_subtract_fee_outputs(params.m_subtract_fee_outputs) {} - void Insert(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants, bool positive_only); + void Insert(const COutput& output, size_t ancestors, size_t descendants, bool positive_only); bool EligibleForSpending(const CoinEligibilityFilter& eligibility_filter) const; CAmount GetSelectionAmount() const; }; @@ -198,13 +222,28 @@ struct OutputGroup * @param[in] use_effective_value Whether to use the input's effective value (when true) or the real value (when false). * @return The waste */ -[[nodiscard]] CAmount GetSelectionWaste(const std::set<CInputCoin>& inputs, CAmount change_cost, CAmount target, bool use_effective_value = true); +[[nodiscard]] CAmount GetSelectionWaste(const std::set<COutput>& inputs, CAmount change_cost, CAmount target, bool use_effective_value = true); + + +/** Chooose a random change target for each transaction to make it harder to fingerprint the Core + * wallet based on the change output values of transactions it creates. + * The random value is between 50ksat and min(2 * payment_value, 1milsat) + * When payment_value <= 25ksat, the value is just 50ksat. + * + * Making change amounts similar to the payment value may help disguise which output(s) are payments + * are which ones are change. Using double the payment value may increase the number of inputs + * needed (and thus be more expensive in fees), but breaks analysis techniques which assume the + * coins selected are just sufficient to cover the payment amount ("unnecessary input" heuristic). + * + * @param[in] payment_value Average payment value of the transaction output(s). + */ +[[nodiscard]] CAmount GenerateChangeTarget(CAmount payment_value, FastRandomContext& rng); struct SelectionResult { private: /** Set of inputs selected by the algorithm to use in the transaction */ - std::set<CInputCoin> m_selected_inputs; + std::set<COutput> m_selected_inputs; /** The target the algorithm selected for. Note that this may not be equal to the recipient amount as it can include non-input fees */ const CAmount m_target; /** Whether the input values for calculations should be the effective value (true) or normal value (false) */ @@ -230,9 +269,9 @@ public: [[nodiscard]] CAmount GetWaste() const; /** Get m_selected_inputs */ - const std::set<CInputCoin>& GetInputSet() const; - /** Get the vector of CInputCoins that will be used to fill in a CTransaction's vin */ - std::vector<CInputCoin> GetShuffledInputVector() const; + const std::set<COutput>& GetInputSet() const; + /** Get the vector of COutputs that will be used to fill in a CTransaction's vin */ + std::vector<COutput> GetShuffledInputVector() const; bool operator<(SelectionResult other) const; }; @@ -246,10 +285,11 @@ std::optional<SelectionResult> SelectCoinsBnB(std::vector<OutputGroup>& utxo_poo * @param[in] target_value The target value to select for * @returns If successful, a SelectionResult, otherwise, std::nullopt */ -std::optional<SelectionResult> SelectCoinsSRD(const std::vector<OutputGroup>& utxo_pool, CAmount target_value); +std::optional<SelectionResult> SelectCoinsSRD(const std::vector<OutputGroup>& utxo_pool, CAmount target_value, FastRandomContext& rng); // Original coin selection algorithm as a fallback -std::optional<SelectionResult> KnapsackSolver(std::vector<OutputGroup>& groups, const CAmount& nTargetValue); +std::optional<SelectionResult> KnapsackSolver(std::vector<OutputGroup>& groups, const CAmount& nTargetValue, + CAmount change_target, FastRandomContext& rng); } // namespace wallet #endif // BITCOIN_WALLET_COINSELECTION_H diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 0ed2658129..8e79ee678e 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -6,6 +6,7 @@ #include <chainparams.h> #include <fs.h> #include <logging.h> +#include <util/system.h> #include <wallet/db.h> #include <exception> @@ -137,4 +138,13 @@ bool IsSQLiteFile(const fs::path& path) // Check the application id matches our network magic return memcmp(Params().MessageStart(), app_id, 4) == 0; } + +void ReadDatabaseArgs(const ArgsManager& args, DatabaseOptions& options) +{ + // Override current options with args values, if any were specified + options.use_unsafe_sync = args.GetBoolArg("-unsafesqlitesync", options.use_unsafe_sync); + options.use_shared_memory = !args.GetBoolArg("-privdb", !options.use_shared_memory); + options.max_log_mb = args.GetIntArg("-dblogsize", options.max_log_mb); +} + } // namespace wallet diff --git a/src/wallet/db.h b/src/wallet/db.h index 5825b00e3a..f09844c37e 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -16,6 +16,7 @@ #include <optional> #include <string> +class ArgsManager; struct bilingual_str; namespace wallet { @@ -207,7 +208,12 @@ struct DatabaseOptions { std::optional<DatabaseFormat> require_format; uint64_t create_flags = 0; SecureString create_passphrase; - bool verify = true; + + // Specialized options. Not every option is supported by every backend. + bool verify = true; //!< Check data integrity on load. + bool use_unsafe_sync = false; //!< Disable file sync for faster performance. + bool use_shared_memory = false; //!< Let other processes access the database. + int64_t max_log_mb = 100; //!< Max log size to allow before consolidating. }; enum class DatabaseStatus { @@ -227,6 +233,7 @@ enum class DatabaseStatus { /** Recursively list database paths in directory. */ std::vector<fs::path> ListDatabases(const fs::path& path); +void ReadDatabaseArgs(const ArgsManager& args, DatabaseOptions& options); std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); fs::path BDBDataFile(const fs::path& path); diff --git a/src/wallet/dump.cpp b/src/wallet/dump.cpp index 6d8508fc72..d80c3e25b0 100644 --- a/src/wallet/dump.cpp +++ b/src/wallet/dump.cpp @@ -19,10 +19,10 @@ namespace wallet { static const std::string DUMP_MAGIC = "BITCOIN_CORE_WALLET_DUMP"; uint32_t DUMP_VERSION = 1; -bool DumpWallet(CWallet& wallet, bilingual_str& error) +bool DumpWallet(const ArgsManager& args, CWallet& wallet, bilingual_str& error) { // Get the dumpfile - std::string dump_filename = gArgs.GetArg("-dumpfile", ""); + std::string dump_filename = args.GetArg("-dumpfile", ""); if (dump_filename.empty()) { error = _("No dump file provided. To use dump, -dumpfile=<filename> must be provided."); return false; @@ -114,10 +114,10 @@ static void WalletToolReleaseWallet(CWallet* wallet) delete wallet; } -bool CreateFromDump(const std::string& name, const fs::path& wallet_path, bilingual_str& error, std::vector<bilingual_str>& warnings) +bool CreateFromDump(const ArgsManager& args, const std::string& name, const fs::path& wallet_path, bilingual_str& error, std::vector<bilingual_str>& warnings) { // Get the dumpfile - std::string dump_filename = gArgs.GetArg("-dumpfile", ""); + std::string dump_filename = args.GetArg("-dumpfile", ""); if (dump_filename.empty()) { error = _("No dump file provided. To use createfromdump, -dumpfile=<filename> must be provided."); return false; @@ -171,7 +171,7 @@ bool CreateFromDump(const std::string& name, const fs::path& wallet_path, biling return false; } // Get the data file format with format_value as the default - std::string file_format = gArgs.GetArg("-format", format_value); + std::string file_format = args.GetArg("-format", format_value); if (file_format.empty()) { error = _("No wallet file format provided. To use createfromdump, -format=<format> must be provided."); return false; @@ -193,6 +193,7 @@ bool CreateFromDump(const std::string& name, const fs::path& wallet_path, biling DatabaseOptions options; DatabaseStatus status; + ReadDatabaseArgs(args, options); options.require_create = true; options.require_format = data_format; std::unique_ptr<WalletDatabase> database = MakeDatabase(wallet_path, options, status, error); diff --git a/src/wallet/dump.h b/src/wallet/dump.h index a879c4db35..bf683e9843 100644 --- a/src/wallet/dump.h +++ b/src/wallet/dump.h @@ -11,11 +11,12 @@ #include <vector> struct bilingual_str; +class ArgsManager; namespace wallet { class CWallet; -bool DumpWallet(CWallet& wallet, bilingual_str& error); -bool CreateFromDump(const std::string& name, const fs::path& wallet_path, bilingual_str& error, std::vector<bilingual_str>& warnings); +bool DumpWallet(const ArgsManager& args, CWallet& wallet, bilingual_str& error); +bool CreateFromDump(const ArgsManager& args, const std::string& name, const fs::path& wallet_path, bilingual_str& error, std::vector<bilingual_str>& warnings); } // namespace wallet #endif // BITCOIN_WALLET_DUMP_H diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 3552c14160..73042424ad 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -135,7 +135,7 @@ static CFeeRate EstimateFeeRate(const CWallet& wallet, const CWalletTx& wtx, con feerate += std::max(node_incremental_relay_fee, wallet_incremental_relay_fee); // Fee rate must also be at least the wallet's GetMinimumFeeRate - CFeeRate min_feerate(GetMinimumFeeRate(wallet, coin_control, /* feeCalc */ nullptr)); + CFeeRate min_feerate(GetMinimumFeeRate(wallet, coin_control, /*feeCalc=*/nullptr)); // Set the required fee rate for the replacement transaction in coin control. return std::max(feerate, min_feerate); @@ -233,12 +233,6 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo // Write back transaction mtx = CMutableTransaction(*tx_new); - // Mark new tx not replaceable, if requested. - if (!coin_control.m_signal_bip125_rbf.value_or(wallet.m_signal_rbf)) { - for (auto& input : mtx.vin) { - if (input.nSequence < 0xfffffffe) input.nSequence = 0xfffffffe; - } - } return Result::OK; } diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 7a83dbc35d..7e21126298 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -80,9 +80,9 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const argsman.AddArg("-walletrbf", strprintf("Send transactions with full-RBF opt-in enabled (RPC only, default: %u)", DEFAULT_WALLET_RBF), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); #ifdef USE_BDB - argsman.AddArg("-dblogsize=<n>", strprintf("Flush wallet database activity from memory to disk log every <n> megabytes (default: %u)", DEFAULT_WALLET_DBLOGSIZE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); + argsman.AddArg("-dblogsize=<n>", strprintf("Flush wallet database activity from memory to disk log every <n> megabytes (default: %u)", DatabaseOptions().max_log_mb), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); argsman.AddArg("-flushwallet", strprintf("Run a thread to flush wallet periodically (default: %u)", DEFAULT_FLUSHWALLET), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); - argsman.AddArg("-privdb", strprintf("Sets the DB_PRIVATE flag in the wallet db environment (default: %u)", DEFAULT_WALLET_PRIVDB), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); + argsman.AddArg("-privdb", strprintf("Sets the DB_PRIVATE flag in the wallet db environment (default: %u)", !DatabaseOptions().use_shared_memory), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); #else argsman.AddHiddenArgs({"-dblogsize", "-flushwallet", "-privdb"}); #endif diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index 9083c304b2..98e843385c 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -111,6 +111,17 @@ WalletTxOut MakeWalletTxOut(const CWallet& wallet, return result; } +WalletTxOut MakeWalletTxOut(const CWallet& wallet, + const COutput& output) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +{ + WalletTxOut result; + result.txout = output.txout; + result.time = output.time; + result.depth_in_main_chain = output.depth; + result.is_spent = wallet.IsSpent(output.outpoint.hash, output.outpoint.n); + return result; +} + class WalletImpl : public Wallet { public: @@ -419,8 +430,8 @@ public: for (const auto& entry : ListCoins(*m_wallet)) { auto& group = result[entry.first]; for (const auto& coin : entry.second) { - group.emplace_back(COutPoint(coin.tx->GetHash(), coin.i), - MakeWalletTxOut(*m_wallet, *coin.tx, coin.i, coin.nDepth)); + group.emplace_back(coin.outpoint, + MakeWalletTxOut(*m_wallet, coin)); } } return result; @@ -544,6 +555,7 @@ public: std::shared_ptr<CWallet> wallet; DatabaseOptions options; DatabaseStatus status; + ReadDatabaseArgs(*m_context.args, options); options.require_create = true; options.create_flags = wallet_creation_flags; options.create_passphrase = passphrase; @@ -553,6 +565,7 @@ public: { DatabaseOptions options; DatabaseStatus status; + ReadDatabaseArgs(*m_context.args, options); options.require_existing = true; return MakeWallet(m_context, LoadWallet(m_context, name, true /* load_on_start */, options, status, error, warnings)); } diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 633d8c5450..c06513588b 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -57,6 +57,7 @@ bool VerifyWallets(WalletContext& context) if (!args.IsArgSet("wallet")) { DatabaseOptions options; DatabaseStatus status; + ReadDatabaseArgs(args, options); bilingual_str error_string; options.require_existing = true; options.verify = false; @@ -84,6 +85,7 @@ bool VerifyWallets(WalletContext& context) DatabaseOptions options; DatabaseStatus status; + ReadDatabaseArgs(args, options); options.require_existing = true; options.verify = true; bilingual_str error_string; @@ -112,6 +114,7 @@ bool LoadWallets(WalletContext& context) } DatabaseOptions options; DatabaseStatus status; + ReadDatabaseArgs(*context.args, options); options.require_existing = true; options.verify = false; // No need to verify, assuming verified earlier in VerifyWallets() bilingual_str error; @@ -127,6 +130,8 @@ bool LoadWallets(WalletContext& context) chain.initError(error); return false; } + + NotifyWalletLoaded(context, pwallet); AddWallet(context, pwallet); } return true; diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index 1a6f06213c..cddf94aab2 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -325,8 +325,8 @@ Balance GetBalance(const CWallet& wallet, const int min_depth, bool avoid_reuse) const CWalletTx& wtx = entry.second; const bool is_trusted{CachedTxIsTrusted(wallet, wtx, trusted_parents)}; const int tx_depth{wallet.GetTxDepthInMainChain(wtx)}; - const CAmount tx_credit_mine{CachedTxGetAvailableCredit(wallet, wtx, /* fUseCache */ true, ISMINE_SPENDABLE | reuse_filter)}; - const CAmount tx_credit_watchonly{CachedTxGetAvailableCredit(wallet, wtx, /* fUseCache */ true, ISMINE_WATCH_ONLY | reuse_filter)}; + const CAmount tx_credit_mine{CachedTxGetAvailableCredit(wallet, wtx, /*fUseCache=*/true, ISMINE_SPENDABLE | reuse_filter)}; + const CAmount tx_credit_watchonly{CachedTxGetAvailableCredit(wallet, wtx, /*fUseCache=*/true, ISMINE_WATCH_ONLY | reuse_filter)}; if (is_trusted && tx_depth >= min_depth) { ret.m_mine_trusted += tx_credit_mine; ret.m_watchonly_trusted += tx_credit_watchonly; diff --git a/src/wallet/rpc/addresses.cpp b/src/wallet/rpc/addresses.cpp index 51587a64a3..d4f6c9d805 100644 --- a/src/wallet/rpc/addresses.cpp +++ b/src/wallet/rpc/addresses.cpp @@ -239,7 +239,7 @@ RPCHelpMan addmultisigaddress() {RPCResult::Type::STR, "address", "The value of the new multisig address"}, {RPCResult::Type::STR_HEX, "redeemScript", "The string value of the hex-encoded redemption script"}, {RPCResult::Type::STR, "descriptor", "The descriptor for this multisig"}, - {RPCResult::Type::ARR, "warnings", /* optional */ true, "Any warnings resulting from the creation of this multisig", + {RPCResult::Type::ARR, "warnings", /*optional=*/true, "Any warnings resulting from the creation of this multisig", { {RPCResult::Type::STR, "", ""}, }}, @@ -597,7 +597,7 @@ RPCHelpMan getaddressinfo() DescriptorScriptPubKeyMan* desc_spk_man = dynamic_cast<DescriptorScriptPubKeyMan*>(spk_man); if (desc_spk_man) { std::string desc_str; - if (desc_spk_man->GetDescriptorString(desc_str, /* priv */ false)) { + if (desc_spk_man->GetDescriptorString(desc_str, /*priv=*/false)) { ret.pushKV("parent_desc", desc_str); } } diff --git a/src/wallet/rpc/coins.cpp b/src/wallet/rpc/coins.cpp index 035541babd..4eccff3969 100644 --- a/src/wallet/rpc/coins.cpp +++ b/src/wallet/rpc/coins.cpp @@ -112,7 +112,7 @@ RPCHelpMan getreceivedbyaddress() LOCK(pwallet->cs_wallet); - return ValueFromAmount(GetReceived(*pwallet, request.params, /* by_label */ false)); + return ValueFromAmount(GetReceived(*pwallet, request.params, /*by_label=*/false)); }, }; } @@ -153,7 +153,7 @@ RPCHelpMan getreceivedbylabel() LOCK(pwallet->cs_wallet); - return ValueFromAmount(GetReceived(*pwallet, request.params, /* by_label */ true)); + return ValueFromAmount(GetReceived(*pwallet, request.params, /*by_label=*/true)); }, }; } @@ -648,16 +648,16 @@ RPCHelpMan listunspent() for (const COutput& out : vecOutputs) { CTxDestination address; - const CScript& scriptPubKey = out.tx->tx->vout[out.i].scriptPubKey; + const CScript& scriptPubKey = out.txout.scriptPubKey; bool fValidAddress = ExtractDestination(scriptPubKey, address); - bool reused = avoid_reuse && pwallet->IsSpentKey(out.tx->GetHash(), out.i); + bool reused = avoid_reuse && pwallet->IsSpentKey(out.outpoint.hash, out.outpoint.n); if (destinations.size() && (!fValidAddress || !destinations.count(address))) continue; UniValue entry(UniValue::VOBJ); - entry.pushKV("txid", out.tx->GetHash().GetHex()); - entry.pushKV("vout", out.i); + entry.pushKV("txid", out.outpoint.hash.GetHex()); + entry.pushKV("vout", (int)out.outpoint.n); if (fValidAddress) { entry.pushKV("address", EncodeDestination(address)); @@ -702,21 +702,21 @@ RPCHelpMan listunspent() } entry.pushKV("scriptPubKey", HexStr(scriptPubKey)); - entry.pushKV("amount", ValueFromAmount(out.tx->tx->vout[out.i].nValue)); - entry.pushKV("confirmations", out.nDepth); - if (!out.nDepth) { + entry.pushKV("amount", ValueFromAmount(out.txout.nValue)); + entry.pushKV("confirmations", out.depth); + if (!out.depth) { size_t ancestor_count, descendant_count, ancestor_size; CAmount ancestor_fees; - pwallet->chain().getTransactionAncestry(out.tx->GetHash(), ancestor_count, descendant_count, &ancestor_size, &ancestor_fees); + pwallet->chain().getTransactionAncestry(out.outpoint.hash, ancestor_count, descendant_count, &ancestor_size, &ancestor_fees); if (ancestor_count) { entry.pushKV("ancestorcount", uint64_t(ancestor_count)); entry.pushKV("ancestorsize", uint64_t(ancestor_size)); entry.pushKV("ancestorfees", uint64_t(ancestor_fees)); } } - entry.pushKV("spendable", out.fSpendable); - entry.pushKV("solvable", out.fSolvable); - if (out.fSolvable) { + entry.pushKV("spendable", out.spendable); + entry.pushKV("solvable", out.solvable); + if (out.solvable) { std::unique_ptr<SigningProvider> provider = pwallet->GetSolvingProvider(scriptPubKey); if (provider) { auto descriptor = InferDescriptor(scriptPubKey, *provider); @@ -724,7 +724,7 @@ RPCHelpMan listunspent() } } if (avoid_reuse) entry.pushKV("reused", reused); - entry.pushKV("safe", out.fSafe); + entry.pushKV("safe", out.safe); results.push_back(entry); } diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index 433b5a1815..07119133b7 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -9,10 +9,12 @@ #include <rpc/rawtransaction_util.h> #include <rpc/util.h> #include <util/fees.h> +#include <util/rbf.h> #include <util/translation.h> #include <util/vector.h> #include <wallet/coincontrol.h> #include <wallet/feebumper.h> +#include <wallet/fees.h> #include <wallet/rpc/util.h> #include <wallet/spend.h> #include <wallet/wallet.h> @@ -21,7 +23,8 @@ namespace wallet { -static void ParseRecipients(const UniValue& address_amounts, const UniValue& subtract_fee_outputs, std::vector<CRecipient> &recipients) { +static void ParseRecipients(const UniValue& address_amounts, const UniValue& subtract_fee_outputs, std::vector<CRecipient>& recipients) +{ std::set<CTxDestination> destinations; int i = 0; for (const std::string& address: address_amounts.getKeys()) { @@ -51,6 +54,93 @@ static void ParseRecipients(const UniValue& address_amounts, const UniValue& sub } } +static void InterpretFeeEstimationInstructions(const UniValue& conf_target, const UniValue& estimate_mode, const UniValue& fee_rate, UniValue& options) +{ + if (options.exists("conf_target") || options.exists("estimate_mode")) { + if (!conf_target.isNull() || !estimate_mode.isNull()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Pass conf_target and estimate_mode either as arguments or in the options object, but not both"); + } + } else { + options.pushKV("conf_target", conf_target); + options.pushKV("estimate_mode", estimate_mode); + } + if (options.exists("fee_rate")) { + if (!fee_rate.isNull()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Pass the fee_rate either as an argument, or in the options object, but not both"); + } + } else { + options.pushKV("fee_rate", fee_rate); + } + if (!options["conf_target"].isNull() && (options["estimate_mode"].isNull() || (options["estimate_mode"].get_str() == "unset"))) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Specify estimate_mode"); + } +} + +static UniValue FinishTransaction(const std::shared_ptr<CWallet> pwallet, const UniValue& options, const CMutableTransaction& rawTx) +{ + // Make a blank psbt + PartiallySignedTransaction psbtx(rawTx); + + // First fill transaction with our data without signing, + // so external signers are not asked sign more than once. + bool complete; + pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, false, true); + const TransactionError err{pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, true, false)}; + if (err != TransactionError::OK) { + throw JSONRPCTransactionError(err); + } + + CMutableTransaction mtx; + complete = FinalizeAndExtractPSBT(psbtx, mtx); + + UniValue result(UniValue::VOBJ); + + const bool psbt_opt_in{options.exists("psbt") && options["psbt"].get_bool()}; + bool add_to_wallet{options.exists("add_to_wallet") ? options["add_to_wallet"].get_bool() : true}; + if (psbt_opt_in || !complete || !add_to_wallet) { + // Serialize the PSBT + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << psbtx; + result.pushKV("psbt", EncodeBase64(ssTx.str())); + } + + if (complete) { + std::string hex{EncodeHexTx(CTransaction(mtx))}; + CTransactionRef tx(MakeTransactionRef(std::move(mtx))); + result.pushKV("txid", tx->GetHash().GetHex()); + if (add_to_wallet && !psbt_opt_in) { + pwallet->CommitTransaction(tx, {}, /*orderForm=*/{}); + } else { + result.pushKV("hex", hex); + } + } + result.pushKV("complete", complete); + + return result; +} + +static void PreventOutdatedOptions(const UniValue& options) +{ + if (options.exists("feeRate")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Use fee_rate (" + CURRENCY_ATOM + "/vB) instead of feeRate"); + } + if (options.exists("changeAddress")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Use change_address instead of changeAddress"); + } + if (options.exists("changePosition")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Use change_position instead of changePosition"); + } + if (options.exists("includeWatching")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Use include_watching instead of includeWatching"); + } + if (options.exists("lockUnspents")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Use lock_unspents instead of lockUnspents"); + } + if (options.exists("subtractFeeFromOutputs")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Use subtract_fee_from_outputs instead of subtractFeeFromOutputs"); + } +} + UniValue SendMoney(CWallet& wallet, const CCoinControl &coin_control, std::vector<CRecipient> &recipients, mapValue_t map_value, bool verbose) { EnsureWalletIsUnlocked(wallet); @@ -108,7 +198,7 @@ static void SetFeeEstimateMode(const CWallet& wallet, CCoinControl& cc, const Un throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both estimate_mode and fee_rate"); } // Fee rates in sat/vB cannot represent more than 3 significant digits. - cc.m_feerate = CFeeRate{AmountFromValue(fee_rate, /* decimals */ 3)}; + cc.m_feerate = CFeeRate{AmountFromValue(fee_rate, /*decimals=*/3)}; if (override_min_fee) cc.fOverrideFeeRate = true; // Default RBF to true for explicit fee_rate, if unset. if (!cc.m_signal_bip125_rbf) cc.m_signal_bip125_rbf = true; @@ -203,7 +293,7 @@ RPCHelpMan sendtoaddress() // We also enable partial spend avoidance if reuse avoidance is set. coin_control.m_avoid_partial_spends |= coin_control.m_avoid_address_reuse; - SetFeeEstimateMode(*pwallet, coin_control, /* conf_target */ request.params[6], /* estimate_mode */ request.params[7], /* fee_rate */ request.params[9], /* override_min_fee */ false); + SetFeeEstimateMode(*pwallet, coin_control, /*conf_target=*/request.params[6], /*estimate_mode=*/request.params[7], /*fee_rate=*/request.params[9], /*override_min_fee=*/false); EnsureWalletIsUnlocked(*pwallet); @@ -306,7 +396,7 @@ RPCHelpMan sendmany() coin_control.m_signal_bip125_rbf = request.params[5].get_bool(); } - SetFeeEstimateMode(*pwallet, coin_control, /* conf_target */ request.params[6], /* estimate_mode */ request.params[7], /* fee_rate */ request.params[8], /* override_min_fee */ false); + SetFeeEstimateMode(*pwallet, coin_control, /*conf_target=*/request.params[6], /*estimate_mode=*/request.params[7], /*fee_rate=*/request.params[8], /*override_min_fee=*/false); std::vector<CRecipient> recipients; ParseRecipients(sendTo, subtractFeeFromAmount, recipients); @@ -360,31 +450,43 @@ RPCHelpMan settxfee() // Only includes key documentation where the key is snake_case in all RPC methods. MixedCase keys can be added later. -static std::vector<RPCArg> FundTxDoc() +static std::vector<RPCArg> FundTxDoc(bool solving_data = true) { - return { + std::vector<RPCArg> args = { {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks"}, {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, std::string() + "The fee estimate mode, must be one of (case insensitive):\n" " \"" + FeeModes("\"\n\"") + "\""}, - {"replaceable", RPCArg::Type::BOOL, RPCArg::DefaultHint{"wallet default"}, "Marks this transaction as BIP125-replaceable.\n" - "Allows this transaction to be replaced by a transaction with higher fees"}, - {"solving_data", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "Keys and scripts needed for producing a final transaction with a dummy signature.\n" - "Used for fee estimation during coin selection.", - { - {"pubkeys", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Public keys involved in this transaction.", - { - {"pubkey", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A public key"}, - }}, - {"scripts", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Scripts involved in this transaction.", - { - {"script", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A script"}, - }}, - {"descriptors", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Descriptors that provide solving data for this transaction.", - { - {"descriptor", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "A descriptor"}, - }}, - }}, + { + "replaceable", RPCArg::Type::BOOL, RPCArg::DefaultHint{"wallet default"}, "Marks this transaction as BIP125-replaceable.\n" + "Allows this transaction to be replaced by a transaction with higher fees" + }, }; + if (solving_data) { + args.push_back({"solving_data", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "Keys and scripts needed for producing a final transaction with a dummy signature.\n" + "Used for fee estimation during coin selection.", + { + { + "pubkeys", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Public keys involved in this transaction.", + { + {"pubkey", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A public key"}, + } + }, + { + "scripts", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Scripts involved in this transaction.", + { + {"script", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A script"}, + } + }, + { + "descriptors", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Descriptors that provide solving data for this transaction.", + { + {"descriptor", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "A descriptor"}, + } + }, + } + }); + } + return args; } void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, int& change_position, const UniValue& options, CCoinControl& coinControl, bool override_min_fee) @@ -659,7 +761,7 @@ RPCHelpMan fundrawtransaction() {"include_unsafe", RPCArg::Type::BOOL, RPCArg::Default{false}, "Include inputs that are not safe to spend (unconfirmed transactions from outside keys and unconfirmed replacement transactions).\n" "Warning: the resulting transaction may become invalid if one of the unsafe inputs disappears.\n" "If that happens, you will need to fund the transaction with different inputs and republish it."}, - {"changeAddress", RPCArg::Type::STR, RPCArg::DefaultHint{"pool address"}, "The bitcoin address to receive the change"}, + {"changeAddress", RPCArg::Type::STR, RPCArg::DefaultHint{"automatic"}, "The bitcoin address to receive the change"}, {"changePosition", RPCArg::Type::NUM, RPCArg::DefaultHint{"random"}, "The index of the change output"}, {"change_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The output type to use. Only valid if changeAddress is not specified. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, {"includeWatching", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Also select inputs which are watch only.\n" @@ -736,7 +838,7 @@ RPCHelpMan fundrawtransaction() CCoinControl coin_control; // Automatically select (additional) coins. Can be overridden by options.add_inputs. coin_control.m_add_inputs = true; - FundTransaction(*pwallet, tx, fee, change_position, request.params[1], coin_control, /* override_min_fee */ true); + FundTransaction(*pwallet, tx, fee, change_position, request.params[1], coin_control, /*override_min_fee=*/true); UniValue result(UniValue::VOBJ); result.pushKV("hex", EncodeHexTx(CTransaction(tx))); @@ -941,7 +1043,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) if (options.exists("replaceable")) { coin_control.m_signal_bip125_rbf = options["replaceable"].get_bool(); } - SetFeeEstimateMode(*pwallet, coin_control, conf_target, options["estimate_mode"], options["fee_rate"], /* override_min_fee */ false); + SetFeeEstimateMode(*pwallet, coin_control, conf_target, options["estimate_mode"], options["fee_rate"], /*override_min_fee=*/false); } // Make sure the results are valid at least up to the most recent block @@ -1056,7 +1158,7 @@ RPCHelpMan send() "Warning: the resulting transaction may become invalid if one of the unsafe inputs disappears.\n" "If that happens, you will need to fund the transaction with different inputs and republish it."}, {"add_to_wallet", RPCArg::Type::BOOL, RPCArg::Default{true}, "When false, returns a serialized transaction which will not be added to the wallet or broadcast"}, - {"change_address", RPCArg::Type::STR_HEX, RPCArg::DefaultHint{"pool address"}, "The bitcoin address to receive the change"}, + {"change_address", RPCArg::Type::STR, RPCArg::DefaultHint{"automatic"}, "The bitcoin address to receive the change"}, {"change_position", RPCArg::Type::NUM, RPCArg::DefaultHint{"random"}, "The index of the change output"}, {"change_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The output type to use. Only valid if change_address is not specified. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, @@ -1126,102 +1228,249 @@ RPCHelpMan send() if (!pwallet) return NullUniValue; UniValue options{request.params[4].isNull() ? UniValue::VOBJ : request.params[4]}; - if (options.exists("conf_target") || options.exists("estimate_mode")) { - if (!request.params[1].isNull() || !request.params[2].isNull()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Pass conf_target and estimate_mode either as arguments or in the options object, but not both"); - } - } else { - options.pushKV("conf_target", request.params[1]); - options.pushKV("estimate_mode", request.params[2]); - } - if (options.exists("fee_rate")) { - if (!request.params[3].isNull()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Pass the fee_rate either as an argument, or in the options object, but not both"); - } - } else { - options.pushKV("fee_rate", request.params[3]); - } - if (!options["conf_target"].isNull() && (options["estimate_mode"].isNull() || (options["estimate_mode"].get_str() == "unset"))) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Specify estimate_mode"); - } - if (options.exists("feeRate")) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Use fee_rate (" + CURRENCY_ATOM + "/vB) instead of feeRate"); - } - if (options.exists("changeAddress")) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Use change_address"); - } - if (options.exists("changePosition")) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Use change_position"); - } - if (options.exists("includeWatching")) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Use include_watching"); - } - if (options.exists("lockUnspents")) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Use lock_unspents"); - } - if (options.exists("subtractFeeFromOutputs")) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Use subtract_fee_from_outputs"); - } + InterpretFeeEstimationInstructions(/*conf_target=*/request.params[1], /*estimate_mode=*/request.params[2], /*fee_rate=*/request.params[3], options); + PreventOutdatedOptions(options); - const bool psbt_opt_in = options.exists("psbt") && options["psbt"].get_bool(); CAmount fee; int change_position; - bool rbf = pwallet->m_signal_rbf; - if (options.exists("replaceable")) { - rbf = options["replaceable"].get_bool(); - } + bool rbf{options.exists("replaceable") ? options["replaceable"].get_bool() : pwallet->m_signal_rbf}; CMutableTransaction rawTx = ConstructTransaction(options["inputs"], request.params[0], options["locktime"], rbf); CCoinControl coin_control; // Automatically select coins, unless at least one is manually selected. Can // be overridden by options.add_inputs. coin_control.m_add_inputs = rawTx.vin.size() == 0; SetOptionsInputWeights(options["inputs"], options); - FundTransaction(*pwallet, rawTx, fee, change_position, options, coin_control, /* override_min_fee */ false); + FundTransaction(*pwallet, rawTx, fee, change_position, options, coin_control, /*override_min_fee=*/false); - bool add_to_wallet = true; - if (options.exists("add_to_wallet")) { - add_to_wallet = options["add_to_wallet"].get_bool(); + return FinishTransaction(pwallet, options, rawTx); + } + }; +} + +RPCHelpMan sendall() +{ + return RPCHelpMan{"sendall", + "EXPERIMENTAL warning: this call may be changed in future releases.\n" + "\nSpend the value of all (or specific) confirmed UTXOs in the wallet to one or more recipients.\n" + "Unconfirmed inbound UTXOs and locked UTXOs will not be spent. Sendall will respect the avoid_reuse wallet flag.\n" + "If your wallet contains many small inputs, either because it received tiny payments or as a result of accumulating change, consider using `send_max` to exclude inputs that are worth less than the fees needed to spend them.\n", + { + {"recipients", RPCArg::Type::ARR, RPCArg::Optional::NO, "The sendall destinations. Each address may only appear once.\n" + "Optionally some recipients can be specified with an amount to perform payments, but at least one address must appear without a specified amount.\n", + { + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "A bitcoin address which receives an equal share of the unspecified amount."}, + {"", RPCArg::Type::OBJ_USER_KEYS, RPCArg::Optional::OMITTED, "", + { + {"address", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "A key-value pair. The key (string) is the bitcoin address, the value (float or string) is the amount in " + CURRENCY_UNIT + ""}, + }, + }, + }, + }, + {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks"}, + {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, std::string() + "The fee estimate mode, must be one of (case insensitive):\n" + " \"" + FeeModes("\"\n\"") + "\""}, + {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, + { + "options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", + Cat<std::vector<RPCArg>>( + { + {"add_to_wallet", RPCArg::Type::BOOL, RPCArg::Default{true}, "When false, returns the serialized transaction without broadcasting or adding it to the wallet"}, + {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, + {"include_watching", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Also select inputs which are watch-only.\n" + "Only solvable inputs can be used. Watch-only destinations are solvable if the public key and/or output script was imported,\n" + "e.g. with 'importpubkey' or 'importmulti' with the 'pubkeys' or 'desc' field."}, + {"inputs", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Use exactly the specified inputs to build the transaction. Specifying inputs is incompatible with send_max. A JSON array of JSON objects", + { + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id"}, + {"vout", RPCArg::Type::NUM, RPCArg::Optional::NO, "The output number"}, + {"sequence", RPCArg::Type::NUM, RPCArg::Optional::NO, "The sequence number"}, + }, + }, + {"locktime", RPCArg::Type::NUM, RPCArg::Default{0}, "Raw locktime. Non-0 value also locktime-activates inputs"}, + {"lock_unspents", RPCArg::Type::BOOL, RPCArg::Default{false}, "Lock selected unspent outputs"}, + {"psbt", RPCArg::Type::BOOL, RPCArg::DefaultHint{"automatic"}, "Always return a PSBT, implies add_to_wallet=false."}, + {"send_max", RPCArg::Type::BOOL, RPCArg::Default{false}, "When true, only use UTXOs that can pay for their own fees to maximize the output amount. When 'false' (default), no UTXO is left behind. send_max is incompatible with providing specific inputs."}, + }, + FundTxDoc() + ), + "options" + }, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::BOOL, "complete", "If the transaction has a complete set of signatures"}, + {RPCResult::Type::STR_HEX, "txid", /*optional=*/true, "The transaction id for the send. Only 1 transaction is created regardless of the number of addresses."}, + {RPCResult::Type::STR_HEX, "hex", /*optional=*/true, "If add_to_wallet is false, the hex-encoded raw transaction with signature(s)"}, + {RPCResult::Type::STR, "psbt", /*optional=*/true, "If more signatures are needed, or if add_to_wallet is false, the base64-encoded (partially) signed transaction"} + } + }, + RPCExamples{"" + "\nSpend all UTXOs from the wallet with a fee rate of 1Â " + CURRENCY_ATOM + "/vB using named arguments\n" + + HelpExampleCli("-named sendall", "recipients='[\"" + EXAMPLE_ADDRESS[0] + "\"]' fee_rate=1\n") + + "Spend all UTXOs with a fee rate of 1.1 " + CURRENCY_ATOM + "/vB using positional arguments\n" + + HelpExampleCli("sendall", "'[\"" + EXAMPLE_ADDRESS[0] + "\"]' null \"unset\" 1.1\n") + + "Spend all UTXOs split into equal amounts to two addresses with a fee rate of 1.5 " + CURRENCY_ATOM + "/vB using the options argument\n" + + HelpExampleCli("sendall", "'[\"" + EXAMPLE_ADDRESS[0] + "\", \"" + EXAMPLE_ADDRESS[1] + "\"]' null \"unset\" null '{\"fee_rate\": 1.5}'\n") + + "Leave dust UTXOs in wallet, spend only UTXOs with positive effective value with a fee rate of 10 " + CURRENCY_ATOM + "/vB using the options argument\n" + + HelpExampleCli("sendall", "'[\"" + EXAMPLE_ADDRESS[0] + "\"]' null \"unset\" null '{\"fee_rate\": 10, \"send_max\": true}'\n") + + "Spend all UTXOs with a fee rate of 1.3 " + CURRENCY_ATOM + "/vB using named arguments and sending a 0.25 " + CURRENCY_UNIT + " to another recipient\n" + + HelpExampleCli("-named sendall", "recipients='[{\"" + EXAMPLE_ADDRESS[1] + "\": 0.25}, \""+ EXAMPLE_ADDRESS[0] + "\"]' fee_rate=1.3\n") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + RPCTypeCheck(request.params, { + UniValue::VARR, // recipients + UniValue::VNUM, // conf_target + UniValue::VSTR, // estimate_mode + UniValueType(), // fee_rate, will be checked by AmountFromValue() in SetFeeEstimateMode() + UniValue::VOBJ, // options + }, true + ); + + std::shared_ptr<CWallet> const pwallet{GetWalletForJSONRPCRequest(request)}; + if (!pwallet) return NullUniValue; + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + + UniValue options{request.params[4].isNull() ? UniValue::VOBJ : request.params[4]}; + InterpretFeeEstimationInstructions(/*conf_target=*/request.params[1], /*estimate_mode=*/request.params[2], /*fee_rate=*/request.params[3], options); + PreventOutdatedOptions(options); + + + std::set<std::string> addresses_without_amount; + UniValue recipient_key_value_pairs(UniValue::VARR); + const UniValue& recipients{request.params[0]}; + for (unsigned int i = 0; i < recipients.size(); ++i) { + const UniValue& recipient{recipients[i]}; + if (recipient.isStr()) { + UniValue rkvp(UniValue::VOBJ); + rkvp.pushKV(recipient.get_str(), 0); + recipient_key_value_pairs.push_back(rkvp); + addresses_without_amount.insert(recipient.get_str()); + } else { + recipient_key_value_pairs.push_back(recipient); + } + } + + if (addresses_without_amount.size() == 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Must provide at least one address without a specified amount"); } - // Make a blank psbt - PartiallySignedTransaction psbtx(rawTx); + CCoinControl coin_control; + + SetFeeEstimateMode(*pwallet, coin_control, options["conf_target"], options["estimate_mode"], options["fee_rate"], /*override_min_fee=*/false); - // First fill transaction with our data without signing, - // so external signers are not asked sign more than once. - bool complete; - pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, false, true); - const TransactionError err = pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, true, false); - if (err != TransactionError::OK) { - throw JSONRPCTransactionError(err); + coin_control.fAllowWatchOnly = ParseIncludeWatchonly(options["include_watching"], *pwallet); + + const bool rbf{options.exists("replaceable") ? options["replaceable"].get_bool() : pwallet->m_signal_rbf}; + + FeeCalculation fee_calc_out; + CFeeRate fee_rate{GetMinimumFeeRate(*pwallet, coin_control, &fee_calc_out)}; + // Do not, ever, assume that it's fine to change the fee rate if the user has explicitly + // provided one + if (coin_control.m_feerate && fee_rate > *coin_control.m_feerate) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Fee rate (%s) is lower than the minimum fee rate setting (%s)", coin_control.m_feerate->ToString(FeeEstimateMode::SAT_VB), fee_rate.ToString(FeeEstimateMode::SAT_VB))); + } + if (fee_calc_out.reason == FeeReason::FALLBACK && !pwallet->m_allow_fallback_fee) { + // eventually allow a fallback fee + throw JSONRPCError(RPC_WALLET_ERROR, "Fee estimation failed. Fallbackfee is disabled. Wait a few blocks or enable -fallbackfee."); } - CMutableTransaction mtx; - complete = FinalizeAndExtractPSBT(psbtx, mtx); + CMutableTransaction rawTx{ConstructTransaction(options["inputs"], recipient_key_value_pairs, options["locktime"], rbf)}; + LOCK(pwallet->cs_wallet); + std::vector<COutput> all_the_utxos; + + CAmount total_input_value(0); + bool send_max{options.exists("send_max") ? options["send_max"].get_bool() : false}; + if (options.exists("inputs") && options.exists("send_max")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot combine send_max with specific inputs."); + } else if (options.exists("inputs")) { + for (const CTxIn& input : rawTx.vin) { + if (pwallet->IsSpent(input.prevout.hash, input.prevout.n)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Input not available. UTXO (%s:%d) was already spent.", input.prevout.hash.ToString(), input.prevout.n)); + } + const CWalletTx* tx{pwallet->GetWalletTx(input.prevout.hash)}; + if (!tx || pwallet->IsMine(tx->tx->vout[input.prevout.n]) != (coin_control.fAllowWatchOnly ? ISMINE_ALL : ISMINE_SPENDABLE)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Input not found. UTXO (%s:%d) is not part of wallet.", input.prevout.hash.ToString(), input.prevout.n)); + } + total_input_value += tx->tx->vout[input.prevout.n].nValue; + } + } else { + AvailableCoins(*pwallet, all_the_utxos, &coin_control, /*nMinimumAmount=*/0); + for (const COutput& output : all_the_utxos) { + CHECK_NONFATAL(output.input_bytes > 0); + if (send_max && fee_rate.GetFee(output.input_bytes) > output.txout.nValue) { + continue; + } + CTxIn input(output.outpoint.hash, output.outpoint.n, CScript(), rbf ? MAX_BIP125_RBF_SEQUENCE : CTxIn::SEQUENCE_FINAL); + rawTx.vin.push_back(input); + total_input_value += output.txout.nValue; + } + } - UniValue result(UniValue::VOBJ); + // estimate final size of tx + const TxSize tx_size{CalculateMaximumSignedTxSize(CTransaction(rawTx), pwallet.get())}; + const CAmount fee_from_size{fee_rate.GetFee(tx_size.vsize)}; + const CAmount effective_value{total_input_value - fee_from_size}; - if (psbt_opt_in || !complete || !add_to_wallet) { - // Serialize the PSBT - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); - ssTx << psbtx; - result.pushKV("psbt", EncodeBase64(ssTx.str())); + if (effective_value <= 0) { + if (send_max) { + throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Total value of UTXO pool too low to pay for transaction, try using lower feerate."); + } else { + throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Total value of UTXO pool too low to pay for transaction. Try using lower feerate or excluding uneconomic UTXOs with 'send_max' option."); + } } - if (complete) { - std::string err_string; - std::string hex = EncodeHexTx(CTransaction(mtx)); - CTransactionRef tx(MakeTransactionRef(std::move(mtx))); - result.pushKV("txid", tx->GetHash().GetHex()); - if (add_to_wallet && !psbt_opt_in) { - pwallet->CommitTransaction(tx, {}, {} /* orderForm */); + CAmount output_amounts_claimed{0}; + for (CTxOut out : rawTx.vout) { + output_amounts_claimed += out.nValue; + } + + if (output_amounts_claimed > total_input_value) { + throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Assigned more value to outputs than available funds."); + } + + const CAmount remainder{effective_value - output_amounts_claimed}; + if (remainder < 0) { + throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Insufficient funds for fees after creating specified outputs."); + } + + const CAmount per_output_without_amount{remainder / (long)addresses_without_amount.size()}; + + bool gave_remaining_to_first{false}; + for (CTxOut& out : rawTx.vout) { + CTxDestination dest; + ExtractDestination(out.scriptPubKey, dest); + std::string addr{EncodeDestination(dest)}; + if (addresses_without_amount.count(addr) > 0) { + out.nValue = per_output_without_amount; + if (!gave_remaining_to_first) { + out.nValue += remainder % addresses_without_amount.size(); + gave_remaining_to_first = true; + } + if (IsDust(out, pwallet->chain().relayDustFee())) { + // Dynamically generated output amount is dust + throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Dynamically assigned remainder results in dust output."); + } } else { - result.pushKV("hex", hex); + if (IsDust(out, pwallet->chain().relayDustFee())) { + // Specified output amount is dust + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Specified output amount to %s is below dust threshold.", addr)); + } + } + } + + const bool lock_unspents{options.exists("lock_unspents") ? options["lock_unspents"].get_bool() : false}; + if (lock_unspents) { + for (const CTxIn& txin : rawTx.vin) { + pwallet->LockCoin(txin.prevout); } } - result.pushKV("complete", complete); - return result; + return FinishTransaction(pwallet, options, rawTx); } }; } @@ -1351,7 +1600,7 @@ RPCHelpMan walletcreatefundedpsbt() {"include_unsafe", RPCArg::Type::BOOL, RPCArg::Default{false}, "Include inputs that are not safe to spend (unconfirmed transactions from outside keys and unconfirmed replacement transactions).\n" "Warning: the resulting transaction may become invalid if one of the unsafe inputs disappears.\n" "If that happens, you will need to fund the transaction with different inputs and republish it."}, - {"changeAddress", RPCArg::Type::STR_HEX, RPCArg::DefaultHint{"pool address"}, "The bitcoin address to receive the change"}, + {"changeAddress", RPCArg::Type::STR, RPCArg::DefaultHint{"automatic"}, "The bitcoin address to receive the change"}, {"changePosition", RPCArg::Type::NUM, RPCArg::DefaultHint{"random"}, "The index of the change output"}, {"change_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The output type to use. Only valid if changeAddress is not specified. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, {"includeWatching", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Also select inputs which are watch only"}, @@ -1418,7 +1667,7 @@ RPCHelpMan walletcreatefundedpsbt() // be overridden by options.add_inputs. coin_control.m_add_inputs = rawTx.vin.size() == 0; SetOptionsInputWeights(request.params[0], options); - FundTransaction(wallet, rawTx, fee, change_position, options, coin_control, /* override_min_fee */ true); + FundTransaction(wallet, rawTx, fee, change_position, options, coin_control, /*override_min_fee=*/true); // Make a blank psbt PartiallySignedTransaction psbtx(rawTx); diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index ad94ce4b32..c87af2ea30 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -800,7 +800,7 @@ RPCHelpMan gettransaction() if (verbose) { UniValue decoded(UniValue::VOBJ); - TxToUniv(*wtx.tx, uint256(), decoded, false); + TxToUniv(*wtx.tx, /*block_hash=*/uint256(), /*entry=*/decoded, /*include_hex=*/false); entry.pushKV("decoded", decoded); } diff --git a/src/wallet/rpc/wallet.cpp b/src/wallet/rpc/wallet.cpp index 883a3c102b..4baf16fdcb 100644 --- a/src/wallet/rpc/wallet.cpp +++ b/src/wallet/rpc/wallet.cpp @@ -8,6 +8,7 @@ #include <rpc/server.h> #include <rpc/util.h> #include <util/translation.h> +#include <wallet/context.h> #include <wallet/receive.h> #include <wallet/rpc/wallet.h> #include <wallet/rpc/util.h> @@ -55,7 +56,7 @@ static RPCHelpMan getwalletinfo() { {RPCResult::Type::NUM, "duration", "elapsed seconds since scan start"}, {RPCResult::Type::NUM, "progress", "scanning progress percentage [0.0, 1.0]"}, - }}, + }, /*skip_type_check=*/true}, {RPCResult::Type::BOOL, "descriptors", "whether this wallet uses descriptors for scriptPubKey management"}, {RPCResult::Type::BOOL, "external_signer", "whether this wallet is configured to use an external signer such as a hardware wallet"}, }}, @@ -220,6 +221,7 @@ static RPCHelpMan loadwallet() DatabaseOptions options; DatabaseStatus status; + ReadDatabaseArgs(*context.args, options); options.require_existing = true; bilingual_str error; std::vector<bilingual_str> warnings; @@ -315,7 +317,9 @@ static RPCHelpMan createwallet() {"blank", RPCArg::Type::BOOL, RPCArg::Default{false}, "Create a blank wallet. A blank wallet has no keys or HD seed. One can be set using sethdseed."}, {"passphrase", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "Encrypt the wallet with this passphrase."}, {"avoid_reuse", RPCArg::Type::BOOL, RPCArg::Default{false}, "Keep track of coin reuse, and treat dirty and clean coins differently with privacy considerations in mind."}, - {"descriptors", RPCArg::Type::BOOL, RPCArg::Default{true}, "Create a native descriptor wallet. The wallet will use descriptors internally to handle address creation"}, + {"descriptors", RPCArg::Type::BOOL, RPCArg::Default{true}, "Create a native descriptor wallet. The wallet will use descriptors internally to handle address creation." + " Setting to \"false\" will create a legacy wallet; however, the legacy wallet type is being deprecated and" + " support for creating and opening legacy wallets will be removed in the future."}, {"load_on_startup", RPCArg::Type::BOOL, RPCArg::Optional::OMITTED_NAMED_ARG, "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, {"external_signer", RPCArg::Type::BOOL, RPCArg::Default{false}, "Use an external signer such as a hardware wallet. Requires -signer to be configured. Wallet creation will fail if keys cannot be fetched. Requires disable_private_keys and descriptors set to true."}, }, @@ -379,6 +383,7 @@ static RPCHelpMan createwallet() DatabaseOptions options; DatabaseStatus status; + ReadDatabaseArgs(*context.args, options); options.require_create = true; options.create_flags = flags; options.create_passphrase = passphrase; @@ -639,6 +644,7 @@ RPCHelpMan fundrawtransaction(); RPCHelpMan bumpfee(); RPCHelpMan psbtbumpfee(); RPCHelpMan send(); +RPCHelpMan sendall(); RPCHelpMan walletprocesspsbt(); RPCHelpMan walletcreatefundedpsbt(); RPCHelpMan signrawtransactionwithwallet(); @@ -718,6 +724,7 @@ static const CRPCCommand commands[] = { "wallet", &setwalletflag, }, { "wallet", &signmessage, }, { "wallet", &signrawtransactionwithwallet, }, + { "wallet", &sendall, }, { "wallet", &unloadwallet, }, { "wallet", &upgradewallet, }, { "wallet", &walletcreatefundedpsbt, }, diff --git a/src/wallet/salvage.cpp b/src/wallet/salvage.cpp index 1ecc96fe0e..9ba3c7fd2c 100644 --- a/src/wallet/salvage.cpp +++ b/src/wallet/salvage.cpp @@ -23,10 +23,11 @@ static bool KeyFilter(const std::string& type) return WalletBatch::IsKeyType(type) || type == DBKeys::HDCHAIN; } -bool RecoverDatabaseFile(const fs::path& file_path, bilingual_str& error, std::vector<bilingual_str>& warnings) +bool RecoverDatabaseFile(const ArgsManager& args, const fs::path& file_path, bilingual_str& error, std::vector<bilingual_str>& warnings) { DatabaseOptions options; DatabaseStatus status; + ReadDatabaseArgs(args, options); options.require_existing = true; options.verify = false; options.require_format = DatabaseFormat::BERKELEY; diff --git a/src/wallet/salvage.h b/src/wallet/salvage.h index 332aceb262..e4822c3c75 100644 --- a/src/wallet/salvage.h +++ b/src/wallet/salvage.h @@ -9,10 +9,11 @@ #include <fs.h> #include <streams.h> +class ArgsManager; struct bilingual_str; namespace wallet { -bool RecoverDatabaseFile(const fs::path& file_path, bilingual_str& error, std::vector<bilingual_str>& warnings); +bool RecoverDatabaseFile(const ArgsManager& args, const fs::path& file_path, bilingual_str& error, std::vector<bilingual_str>& warnings); } // namespace wallet #endif // BITCOIN_WALLET_SALVAGE_H diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 83eaececc1..9e508f3a32 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -19,6 +19,8 @@ #include <wallet/transaction.h> #include <wallet/wallet.h> +#include <cmath> + using interfaces::FoundBlock; namespace wallet { @@ -29,11 +31,6 @@ int GetTxSpendSize(const CWallet& wallet, const CWalletTx& wtx, unsigned int out return CalculateMaximumSignedInputSize(wtx.tx->vout[out], &wallet, use_max_sig); } -std::string COutput::ToString() const -{ - return strprintf("COutput(%s, %d, %d) [%s]", tx->GetHash().ToString(), i, nDepth, FormatMoney(tx->tx->vout[i].nValue)); -} - int CalculateMaximumSignedInputSize(const CTxOut& txout, const SigningProvider* provider, bool use_max_sig) { CMutableTransaction txn; @@ -158,6 +155,8 @@ void AvailableCoins(const CWallet& wallet, std::vector<COutput>& vCoins, const C continue; } + bool tx_from_me = CachedTxIsFromMe(wallet, wtx, ISMINE_ALL); + for (unsigned int i = 0; i < wtx.tx->vout.size(); i++) { // Only consider selected coins if add_inputs is false if (coinControl && !coinControl->m_add_inputs && !coinControl->IsSelected(COutPoint(entry.first, i))) { @@ -190,8 +189,9 @@ void AvailableCoins(const CWallet& wallet, std::vector<COutput>& vCoins, const C bool solvable = provider ? IsSolvable(*provider, wtx.tx->vout[i].scriptPubKey) : false; bool spendable = ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || (((mine & ISMINE_WATCH_ONLY) != ISMINE_NO) && (coinControl && coinControl->fAllowWatchOnly && solvable)); + int input_bytes = GetTxSpendSize(wallet, wtx, i, (coinControl && coinControl->fAllowWatchOnly)); - vCoins.push_back(COutput(wallet, wtx, i, nDepth, spendable, solvable, safeTx, (coinControl && coinControl->fAllowWatchOnly))); + vCoins.emplace_back(COutPoint(wtx.GetHash(), i), wtx.tx->vout.at(i), nDepth, input_bytes, spendable, solvable, safeTx, wtx.GetTxTime(), tx_from_me); // Checks the sum amount of all UTXO's. if (nMinimumSumAmount != MAX_MONEY) { @@ -218,8 +218,8 @@ CAmount GetAvailableBalance(const CWallet& wallet, const CCoinControl* coinContr std::vector<COutput> vCoins; AvailableCoins(wallet, vCoins, coinControl); for (const COutput& out : vCoins) { - if (out.fSpendable) { - balance += out.tx->tx->vout[out.i].nValue; + if (out.spendable) { + balance += out.txout.nValue; } } return balance; @@ -243,6 +243,12 @@ const CTxOut& FindNonChangeParentOutput(const CWallet& wallet, const CTransactio return ptx->vout[n]; } +const CTxOut& FindNonChangeParentOutput(const CWallet& wallet, const COutPoint& outpoint) +{ + AssertLockHeld(wallet.cs_wallet); + return FindNonChangeParentOutput(wallet, *wallet.GetWalletTx(outpoint.hash)->tx, outpoint.n); +} + std::map<CTxDestination, std::vector<COutput>> ListCoins(const CWallet& wallet) { AssertLockHeld(wallet.cs_wallet); @@ -254,8 +260,8 @@ std::map<CTxDestination, std::vector<COutput>> ListCoins(const CWallet& wallet) for (const COutput& coin : availableCoins) { CTxDestination address; - if ((coin.fSpendable || (wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && coin.fSolvable)) && - ExtractDestination(FindNonChangeParentOutput(wallet, *coin.tx->tx, coin.i).scriptPubKey, address)) { + if ((coin.spendable || (wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && coin.solvable)) && + ExtractDestination(FindNonChangeParentOutput(wallet, coin.outpoint).scriptPubKey, address)) { result[address].emplace_back(std::move(coin)); } } @@ -268,14 +274,15 @@ std::map<CTxDestination, std::vector<COutput>> ListCoins(const CWallet& wallet) for (const COutPoint& output : lockedCoins) { auto it = wallet.mapWallet.find(output.hash); if (it != wallet.mapWallet.end()) { - int depth = wallet.GetTxDepthInMainChain(it->second); - if (depth >= 0 && output.n < it->second.tx->vout.size() && - wallet.IsMine(it->second.tx->vout[output.n]) == is_mine_filter + const auto& wtx = it->second; + int depth = wallet.GetTxDepthInMainChain(wtx); + if (depth >= 0 && output.n < wtx.tx->vout.size() && + wallet.IsMine(wtx.tx->vout[output.n]) == is_mine_filter ) { CTxDestination address; - if (ExtractDestination(FindNonChangeParentOutput(wallet, *it->second.tx, output.n).scriptPubKey, address)) { + if (ExtractDestination(FindNonChangeParentOutput(wallet, *wtx.tx, output.n).scriptPubKey, address)) { result[address].emplace_back( - wallet, it->second, output.n, depth, true /* spendable */, true /* solvable */, false /* safe */); + COutPoint(wtx.GetHash(), output.n), wtx.tx->vout.at(output.n), depth, GetTxSpendSize(wallet, wtx, output.n), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ false, wtx.GetTxTime(), CachedTxIsFromMe(wallet, wtx, ISMINE_ALL)); } } } @@ -292,15 +299,14 @@ std::vector<OutputGroup> GroupOutputs(const CWallet& wallet, const std::vector<C // Allowing partial spends means no grouping. Each COutput gets its own OutputGroup. for (const COutput& output : outputs) { // Skip outputs we cannot spend - if (!output.fSpendable) continue; + if (!output.spendable) continue; size_t ancestors, descendants; - wallet.chain().getTransactionAncestry(output.tx->GetHash(), ancestors, descendants); - CInputCoin input_coin = output.GetInputCoin(); + wallet.chain().getTransactionAncestry(output.outpoint.hash, ancestors, descendants); // Make an OutputGroup containing just this output OutputGroup group{coin_sel_params}; - group.Insert(input_coin, output.nDepth, CachedTxIsFromMe(wallet, *output.tx, ISMINE_ALL), ancestors, descendants, positive_only); + group.Insert(output, ancestors, descendants, positive_only); // Check the OutputGroup's eligibility. Only add the eligible ones. if (positive_only && group.GetSelectionAmount() <= 0) continue; @@ -312,18 +318,17 @@ std::vector<OutputGroup> GroupOutputs(const CWallet& wallet, const std::vector<C // We want to combine COutputs that have the same scriptPubKey into single OutputGroups // except when there are more than OUTPUT_GROUP_MAX_ENTRIES COutputs grouped in an OutputGroup. // To do this, we maintain a map where the key is the scriptPubKey and the value is a vector of OutputGroups. - // For each COutput, we check if the scriptPubKey is in the map, and if it is, the COutput's CInputCoin is added + // For each COutput, we check if the scriptPubKey is in the map, and if it is, the COutput is added // to the last OutputGroup in the vector for the scriptPubKey. When the last OutputGroup has - // OUTPUT_GROUP_MAX_ENTRIES CInputCoins, a new OutputGroup is added to the end of the vector. + // OUTPUT_GROUP_MAX_ENTRIES COutputs, a new OutputGroup is added to the end of the vector. std::map<CScript, std::vector<OutputGroup>> spk_to_groups_map; for (const auto& output : outputs) { // Skip outputs we cannot spend - if (!output.fSpendable) continue; + if (!output.spendable) continue; size_t ancestors, descendants; - wallet.chain().getTransactionAncestry(output.tx->GetHash(), ancestors, descendants); - CInputCoin input_coin = output.GetInputCoin(); - CScript spk = input_coin.txout.scriptPubKey; + wallet.chain().getTransactionAncestry(output.outpoint.hash, ancestors, descendants); + CScript spk = output.txout.scriptPubKey; std::vector<OutputGroup>& groups = spk_to_groups_map[spk]; @@ -332,7 +337,7 @@ std::vector<OutputGroup> GroupOutputs(const CWallet& wallet, const std::vector<C groups.emplace_back(coin_sel_params); } - // Get the last OutputGroup in the vector so that we can add the CInputCoin to it + // Get the last OutputGroup in the vector so that we can add the COutput to it // A pointer is used here so that group can be reassigned later if it is full. OutputGroup* group = &groups.back(); @@ -344,8 +349,8 @@ std::vector<OutputGroup> GroupOutputs(const CWallet& wallet, const std::vector<C group = &groups.back(); } - // Add the input_coin to group - group->Insert(input_coin, output.nDepth, CachedTxIsFromMe(wallet, *output.tx, ISMINE_ALL), ancestors, descendants, positive_only); + // Add the output to group + group->Insert(output, ancestors, descendants, positive_only); } // Now we go through the entire map and pull out the OutputGroups @@ -379,7 +384,6 @@ std::optional<SelectionResult> AttemptSelection(const CWallet& wallet, const CAm // Note that unlike KnapsackSolver, we do not include the fee for creating a change output as BnB will not create a change output. std::vector<OutputGroup> positive_groups = GroupOutputs(wallet, coins, coin_selection_params, eligibility_filter, true /* positive_only */); if (auto bnb_result{SelectCoinsBnB(positive_groups, nTargetValue, coin_selection_params.m_cost_of_change)}) { - bnb_result->ComputeAndSetWaste(CAmount(0)); results.push_back(*bnb_result); } @@ -387,15 +391,18 @@ std::optional<SelectionResult> AttemptSelection(const CWallet& wallet, const CAm std::vector<OutputGroup> all_groups = GroupOutputs(wallet, coins, coin_selection_params, eligibility_filter, false /* positive_only */); // While nTargetValue includes the transaction fees for non-input things, it does not include the fee for creating a change output. // So we need to include that for KnapsackSolver as well, as we are expecting to create a change output. - if (auto knapsack_result{KnapsackSolver(all_groups, nTargetValue + coin_selection_params.m_change_fee)}) { + if (auto knapsack_result{KnapsackSolver(all_groups, nTargetValue + coin_selection_params.m_change_fee, + coin_selection_params.m_min_change_target, coin_selection_params.rng_fast)}) { knapsack_result->ComputeAndSetWaste(coin_selection_params.m_cost_of_change); results.push_back(*knapsack_result); } - // We include the minimum final change for SRD as we do want to avoid making really small change. - // KnapsackSolver does not need this because it includes MIN_CHANGE internally. - const CAmount srd_target = nTargetValue + coin_selection_params.m_change_fee + MIN_FINAL_CHANGE; - if (auto srd_result{SelectCoinsSRD(positive_groups, srd_target)}) { + // Include change for SRD as we want to avoid making really small change if the selection just + // barely meets the target. Just use the lower bound change target instead of the randomly + // generated one, since SRD will result in a random change amount anyway; avoid making the + // target needlessly large. + const CAmount srd_target = nTargetValue + coin_selection_params.m_change_fee + CHANGE_LOWER; + if (auto srd_result{SelectCoinsSRD(positive_groups, srd_target, coin_selection_params.rng_fast)}) { srd_result->ComputeAndSetWaste(coin_selection_params.m_cost_of_change); results.push_back(*srd_result); } @@ -422,11 +429,11 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, const std::vec if (coin_control.HasSelected() && !coin_control.fAllowOtherInputs) { for (const COutput& out : vCoins) { - if (!out.fSpendable) continue; - /* Set depth, from_me, ancestors, and descendants to 0 or false as these don't matter for preset inputs as no actual selection is being done. + if (!out.spendable) continue; + /* Set ancestors and descendants to 0 as these don't matter for preset inputs as no actual selection is being done. * positive_only is set to false because we want to include all preset inputs, even if they are dust. */ - preset_inputs.Insert(out.GetInputCoin(), 0, false, 0, 0, false); + preset_inputs.Insert(out, /*ancestors=*/ 0, /*descendants=*/ 0, /*positive_only=*/ false); } SelectionResult result(nTargetValue); result.AddInput(preset_inputs); @@ -435,7 +442,7 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, const std::vec } // calculate value from preset inputs and store them - std::set<CInputCoin> setPresetCoins; + std::set<COutPoint> preset_coins; std::vector<COutPoint> vPresetInputs; coin_control.ListSelected(vPresetInputs); @@ -456,34 +463,36 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, const std::vec if (!coin_control.GetExternalOutput(outpoint, txout)) { return std::nullopt; } - input_bytes = CalculateMaximumSignedInputSize(txout, &coin_control.m_external_provider, /* use_max_sig */ true); + input_bytes = CalculateMaximumSignedInputSize(txout, &coin_control.m_external_provider, /*use_max_sig=*/true); } // If available, override calculated size with coin control specified size if (coin_control.HasInputWeight(outpoint)) { input_bytes = GetVirtualTransactionSize(coin_control.GetInputWeight(outpoint), 0, 0); } - CInputCoin coin(outpoint, txout, input_bytes); - if (coin.m_input_bytes == -1) { + if (input_bytes == -1) { return std::nullopt; // Not solvable, can't estimate size for fee } - coin.effective_value = coin.txout.nValue - coin_selection_params.m_effective_feerate.GetFee(coin.m_input_bytes); + + /* Set some defaults for depth, spendable, solvable, safe, time, and from_me as these don't matter for preset inputs since no selection is being done. */ + COutput output(outpoint, txout, /*depth=*/ 0, input_bytes, /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, /*time=*/ 0, /*from_me=*/ false); + output.effective_value = output.txout.nValue - coin_selection_params.m_effective_feerate.GetFee(output.input_bytes); if (coin_selection_params.m_subtract_fee_outputs) { - value_to_select -= coin.txout.nValue; + value_to_select -= output.txout.nValue; } else { - value_to_select -= coin.effective_value; + value_to_select -= output.effective_value; } - setPresetCoins.insert(coin); - /* Set depth, from_me, ancestors, and descendants to 0 or false as don't matter for preset inputs as no actual selection is being done. + preset_coins.insert(outpoint); + /* Set ancestors and descendants to 0 as they don't matter for preset inputs since no actual selection is being done. * positive_only is set to false because we want to include all preset inputs, even if they are dust. */ - preset_inputs.Insert(coin, 0, false, 0, 0, false); + preset_inputs.Insert(output, /*ancestors=*/ 0, /*descendants=*/ 0, /*positive_only=*/ false); } // remove preset inputs from vCoins so that Coin Selection doesn't pick them. for (std::vector<COutput>::iterator it = vCoins.begin(); it != vCoins.end() && coin_control.HasSelected();) { - if (setPresetCoins.count(it->GetInputCoin())) + if (preset_coins.count(it->outpoint)) it = vCoins.erase(it); else ++it; @@ -502,7 +511,7 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, const std::vec // Cases where we have 101+ outputs all pointing to the same destination may result in // privacy leaks as they will potentially be deterministically sorted. We solve that by // explicitly shuffling the outputs before processing - Shuffle(vCoins.begin(), vCoins.end(), FastRandomContext()); + Shuffle(vCoins.begin(), vCoins.end(), coin_selection_params.rng_fast); } // Coin Selection attempts to select inputs from a pool of eligible UTXOs to fund the @@ -583,12 +592,14 @@ static bool IsCurrentForAntiFeeSniping(interfaces::Chain& chain, const uint256& } /** - * Return a height-based locktime for new transactions (uses the height of the + * Set a height-based locktime for new transactions (uses the height of the * current chain tip unless we are not synced with the current chain */ -static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, const uint256& block_hash, int block_height) +static void DiscourageFeeSniping(CMutableTransaction& tx, FastRandomContext& rng_fast, + interfaces::Chain& chain, const uint256& block_hash, int block_height) { - uint32_t locktime; + // All inputs must be added by now + assert(!tx.vin.empty()); // Discourage fee sniping. // // For a large miner the value of the transactions in the best block and @@ -610,22 +621,34 @@ static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, const uin // now we ensure code won't be written that makes assumptions about // nLockTime that preclude a fix later. if (IsCurrentForAntiFeeSniping(chain, block_hash)) { - locktime = block_height; + tx.nLockTime = block_height; // Secondly occasionally randomly pick a nLockTime even further back, so // that transactions that are delayed after signing for whatever reason, // e.g. high-latency mix networks and some CoinJoin implementations, have // better privacy. - if (GetRandInt(10) == 0) - locktime = std::max(0, (int)locktime - GetRandInt(100)); + if (rng_fast.randrange(10) == 0) { + tx.nLockTime = std::max(0, int(tx.nLockTime) - int(rng_fast.randrange(100))); + } } else { // If our chain is lagging behind, we can't discourage fee sniping nor help // the privacy of high-latency transactions. To avoid leaking a potentially // unique "nLockTime fingerprint", set nLockTime to a constant. - locktime = 0; + tx.nLockTime = 0; + } + // Sanity check all values + assert(tx.nLockTime < LOCKTIME_THRESHOLD); // Type must be block height + assert(tx.nLockTime <= uint64_t(block_height)); + for (const auto& in : tx.vin) { + // Can not be FINAL for locktime to work + assert(in.nSequence != CTxIn::SEQUENCE_FINAL); + // May be MAX NONFINAL to disable both BIP68 and BIP125 + if (in.nSequence == CTxIn::MAX_SEQUENCE_NONFINAL) continue; + // May be MAX BIP125 to disable BIP68 and enable BIP125 + if (in.nSequence == MAX_BIP125_RBF_SEQUENCE) continue; + // The wallet does not support any other sequence-use right now. + assert(false); } - assert(locktime < LOCKTIME_THRESHOLD); - return locktime; } static bool CreateTransactionInternal( @@ -641,10 +664,10 @@ static bool CreateTransactionInternal( { AssertLockHeld(wallet.cs_wallet); + FastRandomContext rng_fast; CMutableTransaction txNew; // The resulting transaction that we make - txNew.nLockTime = GetLocktimeForNewTransaction(wallet.chain(), wallet.GetLastBlockHash(), wallet.GetLastBlockHeight()); - CoinSelectionParams coin_selection_params; // Parameters for coin selection, init with dummy + CoinSelectionParams coin_selection_params{rng_fast}; // Parameters for coin selection, init with dummy coin_selection_params.m_avoid_partial_spends = coin_control.m_avoid_partial_spends; // Set the long term feerate estimate to the wallet's consolidate feerate @@ -662,6 +685,7 @@ static bool CreateTransactionInternal( coin_selection_params.m_subtract_fee_outputs = true; } } + coin_selection_params.m_change_target = GenerateChangeTarget(std::floor(recipients_sum / vecSend.size()), rng_fast); // Create change script that will be used if we need change CScript scriptChange; @@ -759,7 +783,7 @@ static bool CreateTransactionInternal( AvailableCoins(wallet, vAvailableCoins, &coin_control, 1, MAX_MONEY, MAX_MONEY, 0); // Choose coins to use - std::optional<SelectionResult> result = SelectCoins(wallet, vAvailableCoins, /* nTargetValue */ selection_target, coin_control, coin_selection_params); + std::optional<SelectionResult> result = SelectCoins(wallet, vAvailableCoins, /*nTargetValue=*/selection_target, coin_control, coin_selection_params); if (!result) { error = _("Insufficient funds"); return false; @@ -771,10 +795,9 @@ static bool CreateTransactionInternal( assert(change_and_fee >= 0); CTxOut newTxOut(change_and_fee, scriptChange); - if (nChangePosInOut == -1) - { + if (nChangePosInOut == -1) { // Insert change txn at random position: - nChangePosInOut = GetRandInt(txNew.vout.size()+1); + nChangePosInOut = rng_fast.randrange(txNew.vout.size() + 1); } else if ((unsigned int)nChangePosInOut > txNew.vout.size()) { @@ -786,10 +809,10 @@ static bool CreateTransactionInternal( auto change_position = txNew.vout.insert(txNew.vout.begin() + nChangePosInOut, newTxOut); // Shuffle selected coins and fill in final vin - std::vector<CInputCoin> selected_coins = result->GetShuffledInputVector(); + std::vector<COutput> selected_coins = result->GetShuffledInputVector(); - // Note how the sequence number is set to non-maxint so that - // the nLockTime set above actually works. + // The sequence number is set to non-maxint so that DiscourageFeeSniping + // works. // // BIP125 defines opt-in RBF as any nSequence < maxint-1, so // we use the highest possible value in that range (maxint-2) @@ -800,6 +823,7 @@ static bool CreateTransactionInternal( for (const auto& coin : selected_coins) { txNew.vin.push_back(CTxIn(coin.outpoint, CScript(), nSequence)); } + DiscourageFeeSniping(txNew, rng_fast, wallet.chain(), wallet.GetLastBlockHash(), wallet.GetLastBlockHeight()); // Calculate the transaction fee TxSize tx_sizes = CalculateMaximumSignedTxSize(CTransaction(txNew), &wallet, &coin_control); diff --git a/src/wallet/spend.h b/src/wallet/spend.h index 4453fb2762..e43aac5273 100644 --- a/src/wallet/spend.h +++ b/src/wallet/spend.h @@ -11,61 +11,11 @@ #include <wallet/wallet.h> namespace wallet { -/** Get the marginal bytes if spending the specified output from this transaction */ +/** Get the marginal bytes if spending the specified output from this transaction. + * use_max_sig indicates whether to use the maximum sized, 72 byte signature when calculating the + * size of the input spend. This should only be set when watch-only outputs are allowed */ int GetTxSpendSize(const CWallet& wallet, const CWalletTx& wtx, unsigned int out, bool use_max_sig = false); -class COutput -{ -public: - const CWalletTx *tx; - - /** Index in tx->vout. */ - int i; - - /** - * Depth in block chain. - * If > 0: the tx is on chain and has this many confirmations. - * If = 0: the tx is waiting confirmation. - * If < 0: a conflicting tx is on chain and has this many confirmations. */ - int nDepth; - - /** Pre-computed estimated size of this output as a fully-signed input in a transaction. Can be -1 if it could not be calculated */ - int nInputBytes; - - /** Whether we have the private keys to spend this output */ - bool fSpendable; - - /** Whether we know how to spend this output, ignoring the lack of keys */ - bool fSolvable; - - /** Whether to use the maximum sized, 72 byte signature when calculating the size of the input spend. This should only be set when watch-only outputs are allowed */ - bool use_max_sig; - - /** - * Whether this output is considered safe to spend. Unconfirmed transactions - * from outside keys and unconfirmed replacement transactions are considered - * unsafe and will not be used to fund new spending transactions. - */ - bool fSafe; - - COutput(const CWallet& wallet, const CWalletTx& wtx, int iIn, int nDepthIn, bool fSpendableIn, bool fSolvableIn, bool fSafeIn, bool use_max_sig_in = false) - { - tx = &wtx; i = iIn; nDepth = nDepthIn; fSpendable = fSpendableIn; fSolvable = fSolvableIn; fSafe = fSafeIn; nInputBytes = -1; use_max_sig = use_max_sig_in; - // If known and signable by the given wallet, compute nInputBytes - // Failure will keep this value -1 - if (fSpendable) { - nInputBytes = GetTxSpendSize(wallet, wtx, i, use_max_sig); - } - } - - std::string ToString() const; - - inline CInputCoin GetInputCoin() const - { - return CInputCoin(tx->tx, i, nInputBytes); - } -}; - //Get the marginal bytes of spending the specified output int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* pwallet, bool use_max_sig = false); int CalculateMaximumSignedInputSize(const CTxOut& txout, const SigningProvider* pwallet, bool use_max_sig = false); @@ -93,6 +43,7 @@ CAmount GetAvailableBalance(const CWallet& wallet, const CCoinControl* coinContr * Find non-change parent output. */ const CTxOut& FindNonChangeParentOutput(const CWallet& wallet, const CTransaction& tx, int output) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); +const CTxOut& FindNonChangeParentOutput(const CWallet& wallet, const COutPoint& outpoint) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); /** * Return list of available coins and locked coins grouped by non-change output address. diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index 2b2181e70b..3f860289f9 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -37,6 +37,22 @@ static void ErrorLogCallback(void* arg, int code, const char* msg) LogPrintf("SQLite Error. Code: %d. Message: %s\n", code, msg); } +static bool BindBlobToStatement(sqlite3_stmt* stmt, + int index, + Span<const std::byte> blob, + const std::string& description) +{ + int res = sqlite3_bind_blob(stmt, index, blob.data(), blob.size(), SQLITE_STATIC); + if (res != SQLITE_OK) { + LogPrintf("Unable to bind %s to statement: %s\n", description, sqlite3_errstr(res)); + sqlite3_clear_bindings(stmt); + sqlite3_reset(stmt); + return false; + } + + return true; +} + static std::optional<int> ReadPragmaInteger(sqlite3* db, const std::string& key, const std::string& description, bilingual_str& error) { std::string stmt_text = strprintf("PRAGMA %s", key); @@ -67,8 +83,8 @@ static void SetPragma(sqlite3* db, const std::string& key, const std::string& va } } -SQLiteDatabase::SQLiteDatabase(const fs::path& dir_path, const fs::path& file_path, bool mock) - : WalletDatabase(), m_mock(mock), m_dir_path(fs::PathToString(dir_path)), m_file_path(fs::PathToString(file_path)) +SQLiteDatabase::SQLiteDatabase(const fs::path& dir_path, const fs::path& file_path, const DatabaseOptions& options, bool mock) + : WalletDatabase(), m_mock(mock), m_dir_path(fs::PathToString(dir_path)), m_file_path(fs::PathToString(file_path)), m_use_unsafe_sync(options.use_unsafe_sync) { { LOCK(g_sqlite_mutex); @@ -239,7 +255,7 @@ void SQLiteDatabase::Open() // Enable fullfsync for the platforms that use it SetPragma(m_db, "fullfsync", "true", "Failed to enable fullfsync"); - if (gArgs.GetBoolArg("-unsafesqlitesync", false)) { + if (m_use_unsafe_sync) { // Use normal synchronous mode for the journal LogPrintf("WARNING SQLite is configured to not wait for data to be flushed to disk. Data loss and corruption may occur.\n"); SetPragma(m_db, "synchronous", "OFF", "Failed to set synchronous mode to OFF"); @@ -377,14 +393,8 @@ bool SQLiteBatch::ReadKey(CDataStream&& key, CDataStream& value) assert(m_read_stmt); // Bind: leftmost parameter in statement is index 1 - int res = sqlite3_bind_blob(m_read_stmt, 1, key.data(), key.size(), SQLITE_STATIC); - if (res != SQLITE_OK) { - LogPrintf("%s: Unable to bind statement: %s\n", __func__, sqlite3_errstr(res)); - sqlite3_clear_bindings(m_read_stmt); - sqlite3_reset(m_read_stmt); - return false; - } - res = sqlite3_step(m_read_stmt); + if (!BindBlobToStatement(m_read_stmt, 1, key, "key")) return false; + int res = sqlite3_step(m_read_stmt); if (res != SQLITE_ROW) { if (res != SQLITE_DONE) { // SQLITE_DONE means "not found", don't log an error in that case. @@ -418,23 +428,11 @@ bool SQLiteBatch::WriteKey(CDataStream&& key, CDataStream&& value, bool overwrit // Bind: leftmost parameter in statement is index 1 // Insert index 1 is key, 2 is value - int res = sqlite3_bind_blob(stmt, 1, key.data(), key.size(), SQLITE_STATIC); - if (res != SQLITE_OK) { - LogPrintf("%s: Unable to bind key to statement: %s\n", __func__, sqlite3_errstr(res)); - sqlite3_clear_bindings(stmt); - sqlite3_reset(stmt); - return false; - } - res = sqlite3_bind_blob(stmt, 2, value.data(), value.size(), SQLITE_STATIC); - if (res != SQLITE_OK) { - LogPrintf("%s: Unable to bind value to statement: %s\n", __func__, sqlite3_errstr(res)); - sqlite3_clear_bindings(stmt); - sqlite3_reset(stmt); - return false; - } + if (!BindBlobToStatement(stmt, 1, key, "key")) return false; + if (!BindBlobToStatement(stmt, 2, value, "value")) return false; // Execute - res = sqlite3_step(stmt); + int res = sqlite3_step(stmt); sqlite3_clear_bindings(stmt); sqlite3_reset(stmt); if (res != SQLITE_DONE) { @@ -449,16 +447,10 @@ bool SQLiteBatch::EraseKey(CDataStream&& key) assert(m_delete_stmt); // Bind: leftmost parameter in statement is index 1 - int res = sqlite3_bind_blob(m_delete_stmt, 1, key.data(), key.size(), SQLITE_STATIC); - if (res != SQLITE_OK) { - LogPrintf("%s: Unable to bind statement: %s\n", __func__, sqlite3_errstr(res)); - sqlite3_clear_bindings(m_delete_stmt); - sqlite3_reset(m_delete_stmt); - return false; - } + if (!BindBlobToStatement(m_delete_stmt, 1, key, "key")) return false; // Execute - res = sqlite3_step(m_delete_stmt); + int res = sqlite3_step(m_delete_stmt); sqlite3_clear_bindings(m_delete_stmt); sqlite3_reset(m_delete_stmt); if (res != SQLITE_DONE) { @@ -473,18 +465,11 @@ bool SQLiteBatch::HasKey(CDataStream&& key) assert(m_read_stmt); // Bind: leftmost parameter in statement is index 1 - bool ret = false; - int res = sqlite3_bind_blob(m_read_stmt, 1, key.data(), key.size(), SQLITE_STATIC); - if (res == SQLITE_OK) { - res = sqlite3_step(m_read_stmt); - if (res == SQLITE_ROW) { - ret = true; - } - } - + if (!BindBlobToStatement(m_read_stmt, 1, key, "key")) return false; + int res = sqlite3_step(m_read_stmt); sqlite3_clear_bindings(m_read_stmt); sqlite3_reset(m_read_stmt); - return ret; + return res == SQLITE_ROW; } bool SQLiteBatch::StartCursor() @@ -561,7 +546,7 @@ std::unique_ptr<SQLiteDatabase> MakeSQLiteDatabase(const fs::path& path, const D { try { fs::path data_file = SQLiteDataFile(path); - auto db = std::make_unique<SQLiteDatabase>(data_file.parent_path(), data_file); + auto db = std::make_unique<SQLiteDatabase>(data_file.parent_path(), data_file, options); if (options.verify && !db->Verify(error)) { status = DatabaseStatus::FAILED_VERIFY; return nullptr; diff --git a/src/wallet/sqlite.h b/src/wallet/sqlite.h index 3ed598d0d2..47b7ebb0ec 100644 --- a/src/wallet/sqlite.h +++ b/src/wallet/sqlite.h @@ -69,7 +69,7 @@ public: SQLiteDatabase() = delete; /** Create DB handle to real database */ - SQLiteDatabase(const fs::path& dir_path, const fs::path& file_path, bool mock = false); + SQLiteDatabase(const fs::path& dir_path, const fs::path& file_path, const DatabaseOptions& options, bool mock = false); ~SQLiteDatabase(); @@ -113,6 +113,7 @@ public: std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) override; sqlite3* m_db{nullptr}; + bool m_use_unsafe_sync; }; std::unique_ptr<SQLiteDatabase> MakeSQLiteDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index b9f12158ca..2a08c8ab57 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -28,20 +28,20 @@ BOOST_FIXTURE_TEST_SUITE(coinselector_tests, WalletTestingSetup) // we repeat those tests this many times and only complain if all iterations of the test fail #define RANDOM_REPEATS 5 -typedef std::set<CInputCoin> CoinSet; +typedef std::set<COutput> CoinSet; static const CoinEligibilityFilter filter_standard(1, 6, 0); static const CoinEligibilityFilter filter_confirmed(1, 1, 0); static const CoinEligibilityFilter filter_standard_extra(6, 6, 0); static int nextLockTime = 0; -static void add_coin(const CAmount& nValue, int nInput, std::vector<CInputCoin>& set) +static void add_coin(const CAmount& nValue, int nInput, std::vector<COutput>& set) { CMutableTransaction tx; tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; tx.nLockTime = nextLockTime++; // so all transactions get different hashes - set.emplace_back(MakeTransactionRef(tx), nInput); + set.emplace_back(COutPoint(tx.GetHash(), nInput), tx.vout.at(nInput), /*depth=*/ 1, /*input_bytes=*/ -1, /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, /*time=*/ 0, /*from_me=*/ false); } static void add_coin(const CAmount& nValue, int nInput, SelectionResult& result) @@ -50,9 +50,9 @@ static void add_coin(const CAmount& nValue, int nInput, SelectionResult& result) tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; tx.nLockTime = nextLockTime++; // so all transactions get different hashes - CInputCoin coin(MakeTransactionRef(tx), nInput); + COutput output(COutPoint(tx.GetHash(), nInput), tx.vout.at(nInput), /*depth=*/ 1, /*input_bytes=*/ -1, /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, /*time=*/ 0, /*from_me=*/ false); OutputGroup group; - group.Insert(coin, 1, false, 0, 0, true); + group.Insert(output, /*ancestors=*/ 0, /*descendants=*/ 0, /*positive_only=*/ true); result.AddInput(group); } @@ -62,10 +62,10 @@ static void add_coin(const CAmount& nValue, int nInput, CoinSet& set, CAmount fe tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; tx.nLockTime = nextLockTime++; // so all transactions get different hashes - CInputCoin coin(MakeTransactionRef(tx), nInput); + COutput coin(COutPoint(tx.GetHash(), nInput), tx.vout.at(nInput), /*depth=*/ 1, /*input_bytes=*/ -1, /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, /*time=*/ 0, /*from_me=*/ false); coin.effective_value = nValue - fee; - coin.m_fee = fee; - coin.m_long_term_fee = long_term_fee; + coin.fee = fee; + coin.long_term_fee = long_term_fee; set.insert(coin); } @@ -82,24 +82,13 @@ static void add_coin(std::vector<COutput>& coins, CWallet& wallet, const CAmount assert(destination_ok); tx.vout[nInput].scriptPubKey = GetScriptForDestination(dest); } - if (fIsFromMe) { - // IsFromMe() returns (GetDebit() > 0), and GetDebit() is 0 if vin.empty(), - // so stop vin being empty, and cache a non-zero Debit to fake out IsFromMe() - tx.vin.resize(1); - } uint256 txid = tx.GetHash(); LOCK(wallet.cs_wallet); auto ret = wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(txid), std::forward_as_tuple(MakeTransactionRef(std::move(tx)), TxStateInactive{})); assert(ret.second); CWalletTx& wtx = (*ret.first).second; - if (fIsFromMe) - { - wtx.m_amounts[CWalletTx::DEBIT].Set(ISMINE_SPENDABLE, 1); - wtx.m_is_cache_empty = false; - } - COutput output(wallet, wtx, nInput, nAge, true /* spendable */, true /* solvable */, true /* safe */); - coins.push_back(output); + coins.emplace_back(COutPoint(wtx.GetHash(), nInput), wtx.tx->vout.at(nInput), nAge, GetTxSpendSize(wallet, wtx, nInput), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, wtx.GetTxTime(), fIsFromMe); } /** Check if SelectionResult a is equivalent to SelectionResult b. @@ -124,11 +113,14 @@ static bool EquivalentResult(const SelectionResult& a, const SelectionResult& b) /** Check if this selection is equal to another one. Equal means same inputs (i.e same value and prevout) */ static bool EqualResult(const SelectionResult& a, const SelectionResult& b) { - std::pair<CoinSet::iterator, CoinSet::iterator> ret = std::mismatch(a.GetInputSet().begin(), a.GetInputSet().end(), b.GetInputSet().begin()); + std::pair<CoinSet::iterator, CoinSet::iterator> ret = std::mismatch(a.GetInputSet().begin(), a.GetInputSet().end(), b.GetInputSet().begin(), + [](const COutput& a, const COutput& b) { + return a.outpoint == b.outpoint; + }); return ret.first == a.GetInputSet().end() && ret.second == b.GetInputSet().end(); } -static CAmount make_hard_case(int utxos, std::vector<CInputCoin>& utxo_pool) +static CAmount make_hard_case(int utxos, std::vector<COutput>& utxo_pool) { utxo_pool.clear(); CAmount target = 0; @@ -140,34 +132,31 @@ static CAmount make_hard_case(int utxos, std::vector<CInputCoin>& utxo_pool) return target; } -inline std::vector<OutputGroup>& GroupCoins(const std::vector<CInputCoin>& coins) -{ - static std::vector<OutputGroup> static_groups; - static_groups.clear(); - for (auto& coin : coins) { - static_groups.emplace_back(); - static_groups.back().Insert(coin, 0, true, 0, 0, false); - } - return static_groups; -} - inline std::vector<OutputGroup>& GroupCoins(const std::vector<COutput>& coins) { static std::vector<OutputGroup> static_groups; static_groups.clear(); for (auto& coin : coins) { static_groups.emplace_back(); - static_groups.back().Insert(coin.GetInputCoin(), coin.nDepth, coin.tx->m_amounts[CWalletTx::DEBIT].m_cached[ISMINE_SPENDABLE] && coin.tx->m_amounts[CWalletTx::DEBIT].m_value[ISMINE_SPENDABLE] == 1 /* HACK: we can't figure out the is_me flag so we use the conditions defined above; perhaps set safe to false for !fIsFromMe in add_coin() */, 0, 0, false); + static_groups.back().Insert(coin, /*ancestors=*/ 0, /*descendants=*/ 0, /*positive_only=*/ false); } return static_groups; } inline std::vector<OutputGroup>& KnapsackGroupOutputs(const std::vector<COutput>& coins, CWallet& wallet, const CoinEligibilityFilter& filter) { - CoinSelectionParams coin_selection_params(/* change_output_size= */ 0, - /* change_spend_size= */ 0, /* effective_feerate= */ CFeeRate(0), - /* long_term_feerate= */ CFeeRate(0), /* discard_feerate= */ CFeeRate(0), - /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); + FastRandomContext rand{}; + CoinSelectionParams coin_selection_params{ + rand, + /*change_output_size=*/ 0, + /*change_spend_size=*/ 0, + /*min_change_target=*/ CENT, + /*effective_feerate=*/ CFeeRate(0), + /*long_term_feerate=*/ CFeeRate(0), + /*discard_feerate=*/ CFeeRate(0), + /*tx_noinputs_size=*/ 0, + /*avoid_partial=*/ false, + }; static std::vector<OutputGroup> static_groups; static_groups = GroupOutputs(wallet, coins, coin_selection_params, filter, /*positive_only=*/false); return static_groups; @@ -176,8 +165,9 @@ inline std::vector<OutputGroup>& KnapsackGroupOutputs(const std::vector<COutput> // Branch and bound coin selection tests BOOST_AUTO_TEST_CASE(bnb_search_test) { + FastRandomContext rand{}; // Setup - std::vector<CInputCoin> utxo_pool; + std::vector<COutput> utxo_pool; SelectionResult expected_result(CAmount(0)); ///////////////////////// @@ -301,10 +291,17 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) } // Make sure that effective value is working in AttemptSelection when BnB is used - CoinSelectionParams coin_selection_params_bnb(/* change_output_size= */ 0, - /* change_spend_size= */ 0, /* effective_feerate= */ CFeeRate(3000), - /* long_term_feerate= */ CFeeRate(1000), /* discard_feerate= */ CFeeRate(1000), - /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); + CoinSelectionParams coin_selection_params_bnb{ + rand, + /*change_output_size=*/ 0, + /*change_spend_size=*/ 0, + /*min_change_target=*/ 0, + /*effective_feerate=*/ CFeeRate(3000), + /*long_term_feerate=*/ CFeeRate(1000), + /*discard_feerate=*/ CFeeRate(1000), + /*tx_noinputs_size=*/ 0, + /*avoid_partial=*/ false, + }; { std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()); wallet->LoadWallet(); @@ -315,13 +312,13 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) std::vector<COutput> coins; add_coin(coins, *wallet, 1); - coins.at(0).nInputBytes = 40; // Make sure that it has a negative effective value. The next check should assert if this somehow got through. Otherwise it will fail + coins.at(0).input_bytes = 40; // Make sure that it has a negative effective value. The next check should assert if this somehow got through. Otherwise it will fail BOOST_CHECK(!SelectCoinsBnB(GroupCoins(coins), 1 * CENT, coin_selection_params_bnb.m_cost_of_change)); // Test fees subtracted from output: coins.clear(); add_coin(coins, *wallet, 1 * CENT); - coins.at(0).nInputBytes = 40; + coins.at(0).input_bytes = 40; coin_selection_params_bnb.m_subtract_fee_outputs = true; const auto result9 = SelectCoinsBnB(GroupCoins(coins), 1 * CENT, coin_selection_params_bnb.m_cost_of_change); BOOST_CHECK(result9); @@ -342,7 +339,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) add_coin(coins, *wallet, 2 * CENT, 6 * 24, false, 0, true); CCoinControl coin_control; coin_control.fAllowOtherInputs = true; - coin_control.Select(COutPoint(coins.at(0).tx->GetHash(), coins.at(0).i)); + coin_control.Select(coins.at(0).outpoint); coin_selection_params_bnb.m_effective_feerate = CFeeRate(0); const auto result10 = SelectCoins(*wallet, coins, 10 * CENT, coin_control, coin_selection_params_bnb); BOOST_CHECK(result10); @@ -351,6 +348,9 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) BOOST_AUTO_TEST_CASE(knapsack_solver_test) { + FastRandomContext rand{}; + const auto temp1{[&rand](std::vector<OutputGroup>& g, const CAmount& v, CAmount c) { return KnapsackSolver(g, v, c, rand); }}; + const auto KnapsackSolver{temp1}; std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()); wallet->LoadWallet(); LOCK(wallet->cs_wallet); @@ -365,25 +365,25 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) coins.clear(); // with an empty wallet we can't even pay one cent - BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 1 * CENT)); + BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 1 * CENT, CENT)); add_coin(coins, *wallet, 1*CENT, 4); // add a new 1 cent coin // with a new 1 cent coin, we still can't find a mature 1 cent - BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 1 * CENT)); + BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 1 * CENT, CENT)); // but we can find a new 1 cent - const auto result1 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 1 * CENT); + const auto result1 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 1 * CENT, CENT); BOOST_CHECK(result1); BOOST_CHECK_EQUAL(result1->GetSelectedValue(), 1 * CENT); add_coin(coins, *wallet, 2*CENT); // add a mature 2 cent coin // we can't make 3 cents of mature coins - BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 3 * CENT)); + BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 3 * CENT, CENT)); // we can make 3 cents of new coins - const auto result2 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 3 * CENT); + const auto result2 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 3 * CENT, CENT); BOOST_CHECK(result2); BOOST_CHECK_EQUAL(result2->GetSelectedValue(), 3 * CENT); @@ -394,38 +394,38 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // now we have new: 1+10=11 (of which 10 was self-sent), and mature: 2+5+20=27. total = 38 // we can't make 38 cents only if we disallow new coins: - BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 38 * CENT)); + BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 38 * CENT, CENT)); // we can't even make 37 cents if we don't allow new coins even if they're from us - BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard_extra), 38 * CENT)); + BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard_extra), 38 * CENT, CENT)); // but we can make 37 cents if we accept new coins from ourself - const auto result3 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 37 * CENT); + const auto result3 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 37 * CENT, CENT); BOOST_CHECK(result3); BOOST_CHECK_EQUAL(result3->GetSelectedValue(), 37 * CENT); // and we can make 38 cents if we accept all new coins - const auto result4 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 38 * CENT); + const auto result4 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 38 * CENT, CENT); BOOST_CHECK(result4); BOOST_CHECK_EQUAL(result4->GetSelectedValue(), 38 * CENT); // try making 34 cents from 1,2,5,10,20 - we can't do it exactly - const auto result5 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 34 * CENT); + const auto result5 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 34 * CENT, CENT); BOOST_CHECK(result5); BOOST_CHECK_EQUAL(result5->GetSelectedValue(), 35 * CENT); // but 35 cents is closest BOOST_CHECK_EQUAL(result5->GetInputSet().size(), 3U); // the best should be 20+10+5. it's incredibly unlikely the 1 or 2 got included (but possible) // when we try making 7 cents, the smaller coins (1,2,5) are enough. We should see just 2+5 - const auto result6 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 7 * CENT); + const auto result6 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 7 * CENT, CENT); BOOST_CHECK(result6); BOOST_CHECK_EQUAL(result6->GetSelectedValue(), 7 * CENT); BOOST_CHECK_EQUAL(result6->GetInputSet().size(), 2U); // when we try making 8 cents, the smaller coins (1,2,5) are exactly enough. - const auto result7 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 8 * CENT); + const auto result7 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 8 * CENT, CENT); BOOST_CHECK(result7); BOOST_CHECK(result7->GetSelectedValue() == 8 * CENT); BOOST_CHECK_EQUAL(result7->GetInputSet().size(), 3U); // when we try making 9 cents, no subset of smaller coins is enough, and we get the next bigger coin (10) - const auto result8 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 9 * CENT); + const auto result8 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 9 * CENT, CENT); BOOST_CHECK(result8); BOOST_CHECK_EQUAL(result8->GetSelectedValue(), 10 * CENT); BOOST_CHECK_EQUAL(result8->GetInputSet().size(), 1U); @@ -440,12 +440,12 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(coins, *wallet, 30*CENT); // now we have 6+7+8+20+30 = 71 cents total // check that we have 71 and not 72 - const auto result9 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 71 * CENT); + const auto result9 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 71 * CENT, CENT); BOOST_CHECK(result9); - BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 72 * CENT)); + BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 72 * CENT, CENT)); // now try making 16 cents. the best smaller coins can do is 6+7+8 = 21; not as good at the next biggest coin, 20 - const auto result10 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 16 * CENT); + const auto result10 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 16 * CENT, CENT); BOOST_CHECK(result10); BOOST_CHECK_EQUAL(result10->GetSelectedValue(), 20 * CENT); // we should get 20 in one coin BOOST_CHECK_EQUAL(result10->GetInputSet().size(), 1U); @@ -453,7 +453,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(coins, *wallet, 5*CENT); // now we have 5+6+7+8+20+30 = 75 cents total // now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, better than the next biggest coin, 20 - const auto result11 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 16 * CENT); + const auto result11 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 16 * CENT, CENT); BOOST_CHECK(result11); BOOST_CHECK_EQUAL(result11->GetSelectedValue(), 18 * CENT); // we should get 18 in 3 coins BOOST_CHECK_EQUAL(result11->GetInputSet().size(), 3U); @@ -461,13 +461,13 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(coins, *wallet, 18*CENT); // now we have 5+6+7+8+18+20+30 // and now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, the same as the next biggest coin, 18 - const auto result12 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 16 * CENT); + const auto result12 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 16 * CENT, CENT); BOOST_CHECK(result12); BOOST_CHECK_EQUAL(result12->GetSelectedValue(), 18 * CENT); // we should get 18 in 1 coin BOOST_CHECK_EQUAL(result12->GetInputSet().size(), 1U); // because in the event of a tie, the biggest coin wins // now try making 11 cents. we should get 5+6 - const auto result13 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 11 * CENT); + const auto result13 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 11 * CENT, CENT); BOOST_CHECK(result13); BOOST_CHECK_EQUAL(result13->GetSelectedValue(), 11 * CENT); BOOST_CHECK_EQUAL(result13->GetInputSet().size(), 2U); @@ -477,12 +477,12 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(coins, *wallet, 2*COIN); add_coin(coins, *wallet, 3*COIN); add_coin(coins, *wallet, 4*COIN); // now we have 5+6+7+8+18+20+30+100+200+300+400 = 1094 cents - const auto result14 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 95 * CENT); + const auto result14 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 95 * CENT, CENT); BOOST_CHECK(result14); BOOST_CHECK_EQUAL(result14->GetSelectedValue(), 1 * COIN); // we should get 1 BTC in 1 coin BOOST_CHECK_EQUAL(result14->GetInputSet().size(), 1U); - const auto result15 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 195 * CENT); + const auto result15 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 195 * CENT, CENT); BOOST_CHECK(result15); BOOST_CHECK_EQUAL(result15->GetSelectedValue(), 2 * COIN); // we should get 2 BTC in 1 coin BOOST_CHECK_EQUAL(result15->GetInputSet().size(), 1U); @@ -490,34 +490,34 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // empty the wallet and start again, now with fractions of a cent, to test small change avoidance coins.clear(); - add_coin(coins, *wallet, MIN_CHANGE * 1 / 10); - add_coin(coins, *wallet, MIN_CHANGE * 2 / 10); - add_coin(coins, *wallet, MIN_CHANGE * 3 / 10); - add_coin(coins, *wallet, MIN_CHANGE * 4 / 10); - add_coin(coins, *wallet, MIN_CHANGE * 5 / 10); - - // try making 1 * MIN_CHANGE from the 1.5 * MIN_CHANGE - // we'll get change smaller than MIN_CHANGE whatever happens, so can expect MIN_CHANGE exactly - const auto result16 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), MIN_CHANGE); + add_coin(coins, *wallet, CENT * 1 / 10); + add_coin(coins, *wallet, CENT * 2 / 10); + add_coin(coins, *wallet, CENT * 3 / 10); + add_coin(coins, *wallet, CENT * 4 / 10); + add_coin(coins, *wallet, CENT * 5 / 10); + + // try making 1 * CENT from the 1.5 * CENT + // we'll get change smaller than CENT whatever happens, so can expect CENT exactly + const auto result16 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), CENT, CENT); BOOST_CHECK(result16); - BOOST_CHECK_EQUAL(result16->GetSelectedValue(), MIN_CHANGE); + BOOST_CHECK_EQUAL(result16->GetSelectedValue(), CENT); // but if we add a bigger coin, small change is avoided - add_coin(coins, *wallet, 1111*MIN_CHANGE); + add_coin(coins, *wallet, 1111*CENT); // try making 1 from 0.1 + 0.2 + 0.3 + 0.4 + 0.5 + 1111 = 1112.5 - const auto result17 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 1 * MIN_CHANGE); + const auto result17 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 1 * CENT, CENT); BOOST_CHECK(result17); - BOOST_CHECK_EQUAL(result17->GetSelectedValue(), 1 * MIN_CHANGE); // we should get the exact amount + BOOST_CHECK_EQUAL(result17->GetSelectedValue(), 1 * CENT); // we should get the exact amount // if we add more small coins: - add_coin(coins, *wallet, MIN_CHANGE * 6 / 10); - add_coin(coins, *wallet, MIN_CHANGE * 7 / 10); + add_coin(coins, *wallet, CENT * 6 / 10); + add_coin(coins, *wallet, CENT * 7 / 10); - // and try again to make 1.0 * MIN_CHANGE - const auto result18 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 1 * MIN_CHANGE); + // and try again to make 1.0 * CENT + const auto result18 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 1 * CENT, CENT); BOOST_CHECK(result18); - BOOST_CHECK_EQUAL(result18->GetSelectedValue(), 1 * MIN_CHANGE); // we should get the exact amount + BOOST_CHECK_EQUAL(result18->GetSelectedValue(), 1 * CENT); // we should get the exact amount // run the 'mtgox' test (see https://blockexplorer.com/tx/29a3efd3ef04f9153d47a990bd7b048a4b2d213daaa5fb8ed670fb85f13bdbcf) // they tried to consolidate 10 50k coins into one 500k coin, and ended up with 50k in change @@ -525,52 +525,52 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) for (int j = 0; j < 20; j++) add_coin(coins, *wallet, 50000 * COIN); - const auto result19 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 500000 * COIN); + const auto result19 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 500000 * COIN, CENT); BOOST_CHECK(result19); BOOST_CHECK_EQUAL(result19->GetSelectedValue(), 500000 * COIN); // we should get the exact amount BOOST_CHECK_EQUAL(result19->GetInputSet().size(), 10U); // in ten coins - // if there's not enough in the smaller coins to make at least 1 * MIN_CHANGE change (0.5+0.6+0.7 < 1.0+1.0), + // if there's not enough in the smaller coins to make at least 1 * CENT change (0.5+0.6+0.7 < 1.0+1.0), // we need to try finding an exact subset anyway // sometimes it will fail, and so we use the next biggest coin: coins.clear(); - add_coin(coins, *wallet, MIN_CHANGE * 5 / 10); - add_coin(coins, *wallet, MIN_CHANGE * 6 / 10); - add_coin(coins, *wallet, MIN_CHANGE * 7 / 10); - add_coin(coins, *wallet, 1111 * MIN_CHANGE); - const auto result20 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 1 * MIN_CHANGE); + add_coin(coins, *wallet, CENT * 5 / 10); + add_coin(coins, *wallet, CENT * 6 / 10); + add_coin(coins, *wallet, CENT * 7 / 10); + add_coin(coins, *wallet, 1111 * CENT); + const auto result20 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 1 * CENT, CENT); BOOST_CHECK(result20); - BOOST_CHECK_EQUAL(result20->GetSelectedValue(), 1111 * MIN_CHANGE); // we get the bigger coin + BOOST_CHECK_EQUAL(result20->GetSelectedValue(), 1111 * CENT); // we get the bigger coin BOOST_CHECK_EQUAL(result20->GetInputSet().size(), 1U); // but sometimes it's possible, and we use an exact subset (0.4 + 0.6 = 1.0) coins.clear(); - add_coin(coins, *wallet, MIN_CHANGE * 4 / 10); - add_coin(coins, *wallet, MIN_CHANGE * 6 / 10); - add_coin(coins, *wallet, MIN_CHANGE * 8 / 10); - add_coin(coins, *wallet, 1111 * MIN_CHANGE); - const auto result21 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), MIN_CHANGE); + add_coin(coins, *wallet, CENT * 4 / 10); + add_coin(coins, *wallet, CENT * 6 / 10); + add_coin(coins, *wallet, CENT * 8 / 10); + add_coin(coins, *wallet, 1111 * CENT); + const auto result21 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), CENT, CENT); BOOST_CHECK(result21); - BOOST_CHECK_EQUAL(result21->GetSelectedValue(), MIN_CHANGE); // we should get the exact amount + BOOST_CHECK_EQUAL(result21->GetSelectedValue(), CENT); // we should get the exact amount BOOST_CHECK_EQUAL(result21->GetInputSet().size(), 2U); // in two coins 0.4+0.6 // test avoiding small change coins.clear(); - add_coin(coins, *wallet, MIN_CHANGE * 5 / 100); - add_coin(coins, *wallet, MIN_CHANGE * 1); - add_coin(coins, *wallet, MIN_CHANGE * 100); + add_coin(coins, *wallet, CENT * 5 / 100); + add_coin(coins, *wallet, CENT * 1); + add_coin(coins, *wallet, CENT * 100); // trying to make 100.01 from these three coins - const auto result22 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), MIN_CHANGE * 10001 / 100); + const auto result22 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), CENT * 10001 / 100, CENT); BOOST_CHECK(result22); - BOOST_CHECK_EQUAL(result22->GetSelectedValue(), MIN_CHANGE * 10105 / 100); // we should get all coins + BOOST_CHECK_EQUAL(result22->GetSelectedValue(), CENT * 10105 / 100); // we should get all coins BOOST_CHECK_EQUAL(result22->GetInputSet().size(), 3U); // but if we try to make 99.9, we should take the bigger of the two small coins to avoid small change - const auto result23 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), MIN_CHANGE * 9990 / 100); + const auto result23 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), CENT * 9990 / 100, CENT); BOOST_CHECK(result23); - BOOST_CHECK_EQUAL(result23->GetSelectedValue(), 101 * MIN_CHANGE); + BOOST_CHECK_EQUAL(result23->GetSelectedValue(), 101 * CENT); BOOST_CHECK_EQUAL(result23->GetInputSet().size(), 2U); } @@ -583,12 +583,12 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // We only create the wallet once to save time, but we still run the coin selection RUN_TESTS times. for (int i = 0; i < RUN_TESTS; i++) { - const auto result24 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 2000); + const auto result24 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 2000, CENT); BOOST_CHECK(result24); - if (amt - 2000 < MIN_CHANGE) { + if (amt - 2000 < CENT) { // needs more than one input: - uint16_t returnSize = std::ceil((2000.0 + MIN_CHANGE)/amt); + uint16_t returnSize = std::ceil((2000.0 + CENT)/amt); CAmount returnValue = amt * returnSize; BOOST_CHECK_EQUAL(result24->GetSelectedValue(), returnValue); BOOST_CHECK_EQUAL(result24->GetInputSet().size(), returnSize); @@ -610,9 +610,9 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) for (int i = 0; i < RUN_TESTS; i++) { // picking 50 from 100 coins doesn't depend on the shuffle, // but does depend on randomness in the stochastic approximation code - const auto result25 = KnapsackSolver(GroupCoins(coins), 50 * COIN); + const auto result25 = KnapsackSolver(GroupCoins(coins), 50 * COIN, CENT); BOOST_CHECK(result25); - const auto result26 = KnapsackSolver(GroupCoins(coins), 50 * COIN); + const auto result26 = KnapsackSolver(GroupCoins(coins), 50 * COIN, CENT); BOOST_CHECK(result26); BOOST_CHECK(!EqualResult(*result25, *result26)); @@ -623,9 +623,9 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // When choosing 1 from 100 identical coins, 1% of the time, this test will choose the same coin twice // which will cause it to fail. // To avoid that issue, run the test RANDOM_REPEATS times and only complain if all of them fail - const auto result27 = KnapsackSolver(GroupCoins(coins), COIN); + const auto result27 = KnapsackSolver(GroupCoins(coins), COIN, CENT); BOOST_CHECK(result27); - const auto result28 = KnapsackSolver(GroupCoins(coins), COIN); + const auto result28 = KnapsackSolver(GroupCoins(coins), COIN, CENT); BOOST_CHECK(result28); if (EqualResult(*result27, *result28)) fails++; @@ -646,9 +646,9 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) int fails = 0; for (int j = 0; j < RANDOM_REPEATS; j++) { - const auto result29 = KnapsackSolver(GroupCoins(coins), 90 * CENT); + const auto result29 = KnapsackSolver(GroupCoins(coins), 90 * CENT, CENT); BOOST_CHECK(result29); - const auto result30 = KnapsackSolver(GroupCoins(coins), 90 * CENT); + const auto result30 = KnapsackSolver(GroupCoins(coins), 90 * CENT, CENT); BOOST_CHECK(result30); if (EqualResult(*result29, *result30)) fails++; @@ -660,6 +660,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) BOOST_AUTO_TEST_CASE(ApproximateBestSubset) { + FastRandomContext rand{}; std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()); wallet->LoadWallet(); LOCK(wallet->cs_wallet); @@ -673,7 +674,7 @@ BOOST_AUTO_TEST_CASE(ApproximateBestSubset) add_coin(coins, *wallet, 1000 * COIN); add_coin(coins, *wallet, 3 * COIN); - const auto result = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 1003 * COIN); + const auto result = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 1003 * COIN, CENT, rand); BOOST_CHECK(result); BOOST_CHECK_EQUAL(result->GetSelectedValue(), 1003 * COIN); BOOST_CHECK_EQUAL(result->GetInputSet().size(), 2U); @@ -714,10 +715,17 @@ BOOST_AUTO_TEST_CASE(SelectCoins_test) CAmount target = rand.randrange(balance - 1000) + 1000; // Perform selection - CoinSelectionParams cs_params(/* change_output_size= */ 34, - /* change_spend_size= */ 148, /* effective_feerate= */ CFeeRate(0), - /* long_term_feerate= */ CFeeRate(0), /* discard_feerate= */ CFeeRate(0), - /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); + CoinSelectionParams cs_params{ + rand, + /*change_output_size=*/ 34, + /*change_spend_size=*/ 148, + /*min_change_target=*/ CENT, + /*effective_feerate=*/ CFeeRate(0), + /*long_term_feerate=*/ CFeeRate(0), + /*discard_feerate=*/ CFeeRate(0), + /*tx_noinputs_size=*/ 0, + /*avoid_partial=*/ false, + }; CCoinControl cc; const auto result = SelectCoins(*wallet, coins, target, cc, cs_params); BOOST_CHECK(result); diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index 35ae3707f8..fbf1e0efd3 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -19,13 +19,13 @@ static std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& path, s { fs::path data_file = BDBDataFile(path); database_filename = fs::PathToString(data_file.filename()); - return GetBerkeleyEnv(data_file.parent_path()); + return GetBerkeleyEnv(data_file.parent_path(), false); } BOOST_AUTO_TEST_CASE(getwalletenv_file) { std::string test_name = "test_name.dat"; - const fs::path datadir = gArgs.GetDataDirNet(); + const fs::path datadir = m_args.GetDataDirNet(); fs::path file_path = datadir / test_name; std::ofstream f{file_path}; f.close(); @@ -39,7 +39,7 @@ BOOST_AUTO_TEST_CASE(getwalletenv_file) BOOST_AUTO_TEST_CASE(getwalletenv_directory) { std::string expected_name = "wallet.dat"; - const fs::path datadir = gArgs.GetDataDirNet(); + const fs::path datadir = m_args.GetDataDirNet(); std::string filename; std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(datadir, filename); @@ -49,8 +49,8 @@ BOOST_AUTO_TEST_CASE(getwalletenv_directory) BOOST_AUTO_TEST_CASE(getwalletenv_g_dbenvs_multiple) { - fs::path datadir = gArgs.GetDataDirNet() / "1"; - fs::path datadir_2 = gArgs.GetDataDirNet() / "2"; + fs::path datadir = m_args.GetDataDirNet() / "1"; + fs::path datadir_2 = m_args.GetDataDirNet() / "2"; std::string filename; std::shared_ptr<BerkeleyEnvironment> env_1 = GetWalletEnv(datadir, filename); diff --git a/src/wallet/test/fuzz/coinselection.cpp b/src/wallet/test/fuzz/coinselection.cpp new file mode 100644 index 0000000000..2693b68cca --- /dev/null +++ b/src/wallet/test/fuzz/coinselection.cpp @@ -0,0 +1,99 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <policy/feerate.h> +#include <primitives/transaction.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/setup_common.h> +#include <wallet/coinselection.h> + +#include <vector> + +namespace wallet { + +static void AddCoin(const CAmount& value, int n_input, int n_input_bytes, int locktime, std::vector<COutput>& coins) +{ + CMutableTransaction tx; + tx.vout.resize(n_input + 1); + tx.vout[n_input].nValue = value; + tx.nLockTime = locktime; // all transactions get different hashes + coins.emplace_back(COutPoint(tx.GetHash(), n_input), tx.vout.at(n_input), /*depth=*/0, n_input_bytes, /*spendable=*/true, /*solvable=*/true, /*safe=*/true, /*time=*/0, /*from_me=*/true); +} + +// Randomly distribute coins to instances of OutputGroup +static void GroupCoins(FuzzedDataProvider& fuzzed_data_provider, const std::vector<COutput>& coins, const CoinSelectionParams& coin_params, bool positive_only, std::vector<OutputGroup>& output_groups) +{ + auto output_group = OutputGroup(coin_params); + bool valid_outputgroup{false}; + for (auto& coin : coins) { + output_group.Insert(coin, /*ancestors=*/0, /*descendants=*/0, positive_only); + // If positive_only was specified, nothing may have been inserted, leading to an empty outpout group + // that would be invalid for the BnB algorithm + valid_outputgroup = !positive_only || output_group.GetSelectionAmount() > 0; + if (valid_outputgroup && fuzzed_data_provider.ConsumeBool()) { + output_groups.push_back(output_group); + output_group = OutputGroup(coin_params); + valid_outputgroup = false; + } + } + if (valid_outputgroup) output_groups.push_back(output_group); +} + +FUZZ_TARGET(coinselection) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + std::vector<COutput> utxo_pool; + + const CFeeRate long_term_fee_rate{ConsumeMoney(fuzzed_data_provider, /*max=*/COIN)}; + const CFeeRate effective_fee_rate{ConsumeMoney(fuzzed_data_provider, /*max=*/COIN)}; + const CAmount cost_of_change{ConsumeMoney(fuzzed_data_provider, /*max=*/COIN)}; + const CAmount target{fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(1, MAX_MONEY)}; + const bool subtract_fee_outputs{fuzzed_data_provider.ConsumeBool()}; + + FastRandomContext fast_random_context{ConsumeUInt256(fuzzed_data_provider)}; + CoinSelectionParams coin_params{fast_random_context}; + coin_params.m_subtract_fee_outputs = subtract_fee_outputs; + coin_params.m_long_term_feerate = long_term_fee_rate; + coin_params.m_effective_feerate = effective_fee_rate; + + // Create some coins + CAmount total_balance{0}; + int next_locktime{0}; + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) + { + const int n_input{fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 10)}; + const int n_input_bytes{fuzzed_data_provider.ConsumeIntegralInRange<int>(100, 10000)}; + const CAmount amount{fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(1, MAX_MONEY)}; + if (total_balance + amount >= MAX_MONEY) { + break; + } + AddCoin(amount, n_input, n_input_bytes, ++next_locktime, utxo_pool); + total_balance += amount; + } + + std::vector<OutputGroup> group_pos; + GroupCoins(fuzzed_data_provider, utxo_pool, coin_params, /*positive_only=*/true, group_pos); + std::vector<OutputGroup> group_all; + GroupCoins(fuzzed_data_provider, utxo_pool, coin_params, /*positive_only=*/false, group_all); + + // Run coinselection algorithms + const auto result_bnb = SelectCoinsBnB(group_pos, target, cost_of_change); + + auto result_srd = SelectCoinsSRD(group_pos, target, fast_random_context); + if (result_srd) result_srd->ComputeAndSetWaste(cost_of_change); + + CAmount change_target{GenerateChangeTarget(target, fast_random_context)}; + auto result_knapsack = KnapsackSolver(group_all, target, change_target, fast_random_context); + if (result_knapsack) result_knapsack->ComputeAndSetWaste(cost_of_change); + + // If the total balance is sufficient for the target and we are not using + // effective values, Knapsack should always find a solution. + if (total_balance >= target && subtract_fee_outputs) { + assert(result_knapsack); + } +} + +} // namespace wallet diff --git a/src/wallet/test/init_test_fixture.cpp b/src/wallet/test/init_test_fixture.cpp index be38cebafd..34c22a9c0d 100644 --- a/src/wallet/test/init_test_fixture.cpp +++ b/src/wallet/test/init_test_fixture.cpp @@ -15,12 +15,12 @@ namespace wallet { InitWalletDirTestingSetup::InitWalletDirTestingSetup(const std::string& chainName) : BasicTestingSetup(chainName) { - m_wallet_loader = MakeWalletLoader(*m_node.chain, *Assert(m_node.args)); + m_wallet_loader = MakeWalletLoader(*m_node.chain, m_args); std::string sep; sep += fs::path::preferred_separator; - m_datadir = gArgs.GetDataDirNet(); + m_datadir = m_args.GetDataDirNet(); m_cwd = fs::current_path(); m_walletdir_path_cases["default"] = m_datadir / "wallets"; @@ -42,14 +42,11 @@ InitWalletDirTestingSetup::InitWalletDirTestingSetup(const std::string& chainNam InitWalletDirTestingSetup::~InitWalletDirTestingSetup() { - gArgs.LockSettings([&](util::Settings& settings) { - settings.forced_settings.erase("walletdir"); - }); fs::current_path(m_cwd); } void InitWalletDirTestingSetup::SetWalletDir(const fs::path& walletdir_path) { - gArgs.ForceSetArg("-walletdir", fs::PathToString(walletdir_path)); + m_args.ForceSetArg("-walletdir", fs::PathToString(walletdir_path)); } } // namespace wallet diff --git a/src/wallet/test/init_tests.cpp b/src/wallet/test/init_tests.cpp index 7fdecc5642..fb0a07e181 100644 --- a/src/wallet/test/init_tests.cpp +++ b/src/wallet/test/init_tests.cpp @@ -18,7 +18,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_default) SetWalletDir(m_walletdir_path_cases["default"]); bool result = m_wallet_loader->verify(); BOOST_CHECK(result == true); - fs::path walletdir = gArgs.GetPathArg("-walletdir"); + fs::path walletdir = m_args.GetPathArg("-walletdir"); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); BOOST_CHECK_EQUAL(walletdir, expected_path); } @@ -28,7 +28,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_custom) SetWalletDir(m_walletdir_path_cases["custom"]); bool result = m_wallet_loader->verify(); BOOST_CHECK(result == true); - fs::path walletdir = gArgs.GetPathArg("-walletdir"); + fs::path walletdir = m_args.GetPathArg("-walletdir"); fs::path expected_path = fs::canonical(m_walletdir_path_cases["custom"]); BOOST_CHECK_EQUAL(walletdir, expected_path); } @@ -68,7 +68,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing) SetWalletDir(m_walletdir_path_cases["trailing"]); bool result = m_wallet_loader->verify(); BOOST_CHECK(result == true); - fs::path walletdir = gArgs.GetPathArg("-walletdir"); + fs::path walletdir = m_args.GetPathArg("-walletdir"); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); BOOST_CHECK_EQUAL(walletdir, expected_path); } @@ -78,7 +78,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing2) SetWalletDir(m_walletdir_path_cases["trailing2"]); bool result = m_wallet_loader->verify(); BOOST_CHECK(result == true); - fs::path walletdir = gArgs.GetPathArg("-walletdir"); + fs::path walletdir = m_args.GetPathArg("-walletdir"); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); BOOST_CHECK_EQUAL(walletdir, expected_path); } diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index b953f402a2..62053ae8d2 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -68,7 +68,6 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) // Try to sign the mutated input SignatureData sigdata; BOOST_CHECK(m_wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, true, true) != TransactionError::OK); - //BOOST_CHECK(spk_man->FillPSBT(psbtx, PrecomputePSBTData(psbtx), SIGHASH_ALL, true, true) != TransactionError::OK); } BOOST_AUTO_TEST_CASE(parse_hd_keypath) diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index cb006dea3a..38d23b6f91 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -9,6 +9,7 @@ namespace wallet { WalletTestingSetup::WalletTestingSetup(const std::string& chainName) : TestingSetup(chainName), + m_wallet_loader{interfaces::MakeWalletLoader(*m_node.chain, *Assert(m_node.args))}, m_wallet(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()) { m_wallet.LoadWallet(); diff --git a/src/wallet/test/wallet_test_fixture.h b/src/wallet/test/wallet_test_fixture.h index d4b855b145..e0b38a40e7 100644 --- a/src/wallet/test/wallet_test_fixture.h +++ b/src/wallet/test/wallet_test_fixture.h @@ -22,7 +22,7 @@ struct WalletTestingSetup : public TestingSetup { explicit WalletTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); ~WalletTestingSetup(); - std::unique_ptr<interfaces::WalletLoader> m_wallet_loader = interfaces::MakeWalletLoader(*m_node.chain, *Assert(m_node.args)); + std::unique_ptr<interfaces::WalletLoader> m_wallet_loader; CWallet m_wallet; std::unique_ptr<interfaces::Handler> m_chain_notifications_handler; }; diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index c59f7e6f05..683f0eb327 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -54,6 +54,7 @@ static const std::shared_ptr<CWallet> TestLoadWallet(WalletContext& context) std::vector<bilingual_str> warnings; auto database = MakeWalletDatabase("", options, status, error); auto wallet = CWallet::Create(context, "", std::move(database), options.create_flags, error, warnings); + NotifyWalletLoaded(context, wallet); if (context.chain) { wallet->postInitProcess(); } @@ -221,7 +222,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) wallet->SetupLegacyScriptPubKeyMan(); WITH_LOCK(wallet->cs_wallet, wallet->SetLastBlockProcessed(newTip->nHeight, newTip->GetBlockHash())); WalletContext context; - context.args = &gArgs; + context.args = &m_args; AddWallet(context, wallet); UniValue keys; keys.setArray(); @@ -277,12 +278,12 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) SetMockTime(KEY_TIME); m_coinbase_txns.emplace_back(CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); - std::string backup_file = fs::PathToString(gArgs.GetDataDirNet() / "wallet.backup"); + std::string backup_file = fs::PathToString(m_args.GetDataDirNet() / "wallet.backup"); // Import key into wallet and call dumpwallet to create backup file. { WalletContext context; - context.args = &gArgs; + context.args = &m_args; const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); { auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); @@ -310,7 +311,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) wallet->SetupLegacyScriptPubKeyMan(); WalletContext context; - context.args = &gArgs; + context.args = &m_args; JSONRPCRequest request; request.context = &context; request.params.setArray(); @@ -339,7 +340,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) { CWallet wallet(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); - CWalletTx wtx{m_coinbase_txns.back(), TxStateConfirmed{m_node.chainman->ActiveChain().Tip()->GetBlockHash(), m_node.chainman->ActiveChain().Height(), /*position_in_block=*/0}}; + CWalletTx wtx{m_coinbase_txns.back(), TxStateConfirmed{m_node.chainman->ActiveChain().Tip()->GetBlockHash(), m_node.chainman->ActiveChain().Height(), /*index=*/0}}; LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -373,7 +374,7 @@ static int64_t AddTx(ChainstateManager& chainman, CWallet& wallet, uint32_t lock block = &inserted.first->second; block->nTime = blockTime; block->phashBlock = &hash; - state = TxStateConfirmed{hash, block->nHeight, /*position_in_block=*/0}; + state = TxStateConfirmed{hash, block->nHeight, /*index=*/0}; } return wallet.AddToWallet(MakeTransactionRef(tx), state, [&](CWalletTx& wtx, bool /* new_tx */) { // Assign wtx.m_state to simplify test and avoid the need to simulate @@ -540,7 +541,7 @@ public: wallet->SetLastBlockProcessed(wallet->GetLastBlockHeight() + 1, m_node.chainman->ActiveChain().Tip()->GetBlockHash()); auto it = wallet->mapWallet.find(tx->GetHash()); BOOST_CHECK(it != wallet->mapWallet.end()); - it->second.m_state = TxStateConfirmed{m_node.chainman->ActiveChain().Tip()->GetBlockHash(), m_node.chainman->ActiveChain().Height(), /*position_in_block=*/1}; + it->second.m_state = TxStateConfirmed{m_node.chainman->ActiveChain().Tip()->GetBlockHash(), m_node.chainman->ActiveChain().Height(), /*index=*/1}; return it->second; } @@ -588,7 +589,7 @@ BOOST_FIXTURE_TEST_CASE(ListCoinsTest, ListCoinsTestingSetup) for (const auto& group : list) { for (const auto& coin : group.second) { LOCK(wallet->cs_wallet); - wallet->LockCoin(COutPoint(coin.tx->GetHash(), coin.i)); + wallet->LockCoin(coin.outpoint); } } { @@ -716,10 +717,10 @@ BOOST_FIXTURE_TEST_CASE(wallet_descriptor_test, BasicTestingSetup) //! rescanning where new transactions in new blocks could be lost. BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) { - gArgs.ForceSetArg("-unsafesqlitesync", "1"); + m_args.ForceSetArg("-unsafesqlitesync", "1"); // Create new wallet with known key and unload it. WalletContext context; - context.args = &gArgs; + context.args = &m_args; context.chain = m_node.chain.get(); auto wallet = TestLoadWallet(context); CKey key; @@ -812,7 +813,7 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) BOOST_FIXTURE_TEST_CASE(CreateWalletWithoutChain, BasicTestingSetup) { WalletContext context; - context.args = &gArgs; + context.args = &m_args; auto wallet = TestLoadWallet(context); BOOST_CHECK(wallet); UnloadWallet(std::move(wallet)); @@ -820,9 +821,9 @@ BOOST_FIXTURE_TEST_CASE(CreateWalletWithoutChain, BasicTestingSetup) BOOST_FIXTURE_TEST_CASE(ZapSelectTx, TestChain100Setup) { - gArgs.ForceSetArg("-unsafesqlitesync", "1"); + m_args.ForceSetArg("-unsafesqlitesync", "1"); WalletContext context; - context.args = &gArgs; + context.args = &m_args; context.chain = m_node.chain.get(); auto wallet = TestLoadWallet(context); CKey key; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 6cf9f9ce74..2a0653c719 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -167,6 +167,14 @@ std::unique_ptr<interfaces::Handler> HandleLoadWallet(WalletContext& context, Lo return interfaces::MakeHandler([&context, it] { LOCK(context.wallets_mutex); context.wallet_load_fns.erase(it); }); } +void NotifyWalletLoaded(WalletContext& context, const std::shared_ptr<CWallet>& wallet) +{ + LOCK(context.wallets_mutex); + for (auto& load_wallet : context.wallet_load_fns) { + load_wallet(interfaces::MakeWallet(context, wallet)); + } +} + static Mutex g_loading_wallet_mutex; static Mutex g_wallet_release_mutex; static std::condition_variable g_wallet_release_cv; @@ -232,6 +240,8 @@ std::shared_ptr<CWallet> LoadWalletInternal(WalletContext& context, const std::s status = DatabaseStatus::FAILED_LOAD; return nullptr; } + + NotifyWalletLoaded(context, wallet); AddWallet(context, wallet); wallet->postInitProcess(); @@ -348,12 +358,19 @@ std::shared_ptr<CWallet> CreateWallet(WalletContext& context, const std::string& wallet->Lock(); } } + + NotifyWalletLoaded(context, wallet); AddWallet(context, wallet); wallet->postInitProcess(); // Write the wallet settings UpdateWalletSetting(*context.chain, name, load_on_start, warnings); + // Legacy wallets are being deprecated, warn if a newly created wallet is legacy + if (!(wallet_creation_flags & WALLET_FLAG_DESCRIPTORS)) { + warnings.push_back(_("Wallet created successfully. The legacy wallet type is being deprecated and support for creating and opening legacy wallets will be removed in the future.")); + } + status = DatabaseStatus::SUCCESS; return wallet; } @@ -361,6 +378,7 @@ std::shared_ptr<CWallet> CreateWallet(WalletContext& context, const std::string& std::shared_ptr<CWallet> RestoreWallet(WalletContext& context, const fs::path& backup_file, const std::string& wallet_name, std::optional<bool> load_on_start, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings) { DatabaseOptions options; + ReadDatabaseArgs(*context.args, options); options.require_existing = true; if (!fs::exists(backup_file)) { @@ -2899,13 +2917,6 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri } { - LOCK(context.wallets_mutex); - for (auto& load_wallet : context.wallet_load_fns) { - load_wallet(interfaces::MakeWallet(context, walletInstance)); - } - } - - { LOCK(walletInstance->cs_wallet); walletInstance->SetBroadcastTransactions(args.GetBoolArg("-walletbroadcast", DEFAULT_WALLETBROADCAST)); walletInstance->WalletLogPrintf("setKeyPool.size() = %u\n", walletInstance->GetKeyPoolSize()); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index e2c5c69c91..26b7f97b5f 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -20,7 +20,6 @@ #include <util/system.h> #include <util/ui_change_type.h> #include <validationinterface.h> -#include <wallet/coinselection.h> #include <wallet/crypter.h> #include <wallet/scriptpubkeyman.h> #include <wallet/transaction.h> @@ -68,6 +67,7 @@ std::shared_ptr<CWallet> LoadWallet(WalletContext& context, const std::string& n std::shared_ptr<CWallet> CreateWallet(WalletContext& context, const std::string& name, std::optional<bool> load_on_start, DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings); std::shared_ptr<CWallet> RestoreWallet(WalletContext& context, const fs::path& backup_file, const std::string& wallet_name, std::optional<bool> load_on_start, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings); std::unique_ptr<interfaces::Handler> HandleLoadWallet(WalletContext& context, LoadWalletFn load_wallet); +void NotifyWalletLoaded(WalletContext& context, const std::shared_ptr<CWallet>& wallet); std::unique_ptr<WalletDatabase> MakeWalletDatabase(const std::string& name, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); //! -paytxfee default @@ -95,7 +95,7 @@ static const CAmount WALLET_INCREMENTAL_RELAY_FEE = 5000; //! Default for -spendzeroconfchange static const bool DEFAULT_SPEND_ZEROCONF_CHANGE = true; //! Default for -walletrejectlongchains -static const bool DEFAULT_WALLET_REJECT_LONG_CHAINS = false; +static const bool DEFAULT_WALLET_REJECT_LONG_CHAINS{true}; //! -txconfirmtarget default static const unsigned int DEFAULT_TX_CONFIRM_TARGET = 6; //! -walletrbf default @@ -112,7 +112,6 @@ constexpr CAmount HIGH_MAX_TX_FEE{100 * HIGH_TX_FEE_PER_KB}; static constexpr size_t DUMMY_NESTED_P2WPKH_INPUT_SIZE = 91; class CCoinControl; -class COutput; class CWalletTx; class ReserveDestination; diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 6e100524a4..7bbed7973f 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -850,10 +850,10 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) // Set the active ScriptPubKeyMans for (auto spk_man_pair : wss.m_active_external_spks) { - pwallet->LoadActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /* internal */ false); + pwallet->LoadActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /*internal=*/false); } for (auto spk_man_pair : wss.m_active_internal_spks) { - pwallet->LoadActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /* internal */ true); + pwallet->LoadActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /*internal=*/true); } // Set the descriptor caches @@ -1189,10 +1189,11 @@ std::unique_ptr<WalletDatabase> CreateDummyWalletDatabase() /** Return object for accessing temporary in-memory database. */ std::unique_ptr<WalletDatabase> CreateMockWalletDatabase() { + DatabaseOptions options; #ifdef USE_SQLITE - return std::make_unique<SQLiteDatabase>("", "", true); + return std::make_unique<SQLiteDatabase>("", "", options, true); #elif USE_BDB - return std::make_unique<BerkeleyDatabase>(std::make_shared<BerkeleyEnvironment>(), ""); + return std::make_unique<BerkeleyDatabase>(std::make_shared<BerkeleyEnvironment>(), "", options); #endif } } // namespace wallet diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 9cd18dd0a5..769175b5a8 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -140,6 +140,7 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) if (command == "create") { DatabaseOptions options; + ReadDatabaseArgs(args, options); options.require_create = true; // If -legacy is set, use it. Otherwise default to false. bool make_legacy = args.GetBoolArg("-legacy", false); @@ -165,6 +166,7 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) } } else if (command == "info") { DatabaseOptions options; + ReadDatabaseArgs(args, options); options.require_existing = true; const std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, args, options); if (!wallet_instance) return false; @@ -174,7 +176,7 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) #ifdef USE_BDB bilingual_str error; std::vector<bilingual_str> warnings; - bool ret = RecoverDatabaseFile(path, error, warnings); + bool ret = RecoverDatabaseFile(args, path, error, warnings); if (!ret) { for (const auto& warning : warnings) { tfm::format(std::cerr, "%s\n", warning.original); @@ -190,11 +192,12 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) #endif } else if (command == "dump") { DatabaseOptions options; + ReadDatabaseArgs(args, options); options.require_existing = true; const std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, args, options); if (!wallet_instance) return false; bilingual_str error; - bool ret = DumpWallet(*wallet_instance, error); + bool ret = DumpWallet(args, *wallet_instance, error); if (!ret && !error.empty()) { tfm::format(std::cerr, "%s\n", error.original); return ret; @@ -204,7 +207,7 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) } else if (command == "createfromdump") { bilingual_str error; std::vector<bilingual_str> warnings; - bool ret = CreateFromDump(name, path, error, warnings); + bool ret = CreateFromDump(args, name, path, error, warnings); for (const auto& warning : warnings) { tfm::format(std::cout, "%s\n", warning.original); } diff --git a/test/README.md b/test/README.md index 2dc09dfce6..84264277f7 100644 --- a/test/README.md +++ b/test/README.md @@ -107,6 +107,34 @@ how many jobs to run, append `--jobs=n` The individual tests and the test_runner harness have many command-line options. Run `test/functional/test_runner.py -h` to see them all. +#### Speed up test runs with a ramdisk + +If you have available RAM on your system you can create a ramdisk to use as the `cache` and `tmp` directories for the functional tests in order to speed them up. +Speed-up amount varies on each system (and according to your ram speed and other variables), but a 2-3x speed-up is not uncommon. + +To create a 4GB ramdisk on Linux at `/mnt/tmp/`: + +```bash +sudo mkdir -p /mnt/tmp +sudo mount -t tmpfs -o size=4g tmpfs /mnt/tmp/ +``` + +Configure the size of the ramdisk using the `size=` option. +The size of the ramdisk needed is relative to the number of concurrent jobs the test suite runs. +For example running the test suite with `--jobs=100` might need a 4GB ramdisk, but running with `--jobs=32` will only need a 2.5GB ramdisk. + +To use, run the test suite specifying the ramdisk as the `cachedir` and `tmpdir`: + +```bash +test/functional/test_runner.py --cachedir=/mnt/tmp/cache --tmpdir=/mnt/tmp +``` + +Once finished with the tests and the disk, and to free the ram, simply unmount the disk: + +```bash +sudo umount /mnt/tmp +``` + #### Troubleshooting and debugging test failures ##### Resource contention diff --git a/test/functional/feature_coinstatsindex.py b/test/functional/feature_coinstatsindex.py index c70f8a83db..f865661894 100755 --- a/test/functional/feature_coinstatsindex.py +++ b/test/functional/feature_coinstatsindex.py @@ -18,9 +18,6 @@ from test_framework.blocktools import ( ) from test_framework.messages import ( COIN, - COutPoint, - CTransaction, - CTxIn, CTxOut, ) from test_framework.script import ( @@ -33,6 +30,11 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, ) +from test_framework.wallet import ( + MiniWallet, + getnewdestination, +) + class CoinStatsIndexTest(BitcoinTestFramework): def set_test_params(self): @@ -40,16 +42,12 @@ class CoinStatsIndexTest(BitcoinTestFramework): self.num_nodes = 2 self.supports_cli = False self.extra_args = [ - # Explicitly set the output type in order to have consistent tx vsize / fees - # for both legacy and descriptor wallets (disables the change address type detection algorithm) - ["-addresstype=bech32", "-changetype=bech32"], + [], ["-coinstatsindex"] ] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) self._test_coin_stats_index() self._test_use_index_option() self._test_reorg_index() @@ -69,9 +67,8 @@ class CoinStatsIndexTest(BitcoinTestFramework): index_hash_options = ['none', 'muhash'] # Generate a normal transaction and mine it - self.generate(node, COINBASE_MATURITY + 1) - address = self.nodes[0].get_deterministic_priv_key().address - node.sendtoaddress(address=address, amount=10, subtractfeefromamount=True) + self.generate(self.wallet, COINBASE_MATURITY + 1) + self.wallet.send_self_transfer(from_node=node) self.generate(node, 1) self.log.info("Test that gettxoutsetinfo() output is consistent with or without coinstatsindex option") @@ -136,36 +133,31 @@ class CoinStatsIndexTest(BitcoinTestFramework): assert_equal(res5['block_info'], { 'unspendable': 0, 'prevout_spent': 50, - 'new_outputs_ex_coinbase': Decimal('49.99995560'), - 'coinbase': Decimal('50.00004440'), + 'new_outputs_ex_coinbase': Decimal('49.99968800'), + 'coinbase': Decimal('50.00031200'), 'unspendables': { 'genesis_block': 0, 'bip30': 0, 'scripts': 0, - 'unclaimed_rewards': 0 + 'unclaimed_rewards': 0, } }) self.block_sanity_check(res5['block_info']) # Generate and send a normal tx with two outputs - tx1_inputs = [] - tx1_outputs = {self.nodes[0].getnewaddress(): 21, self.nodes[0].getnewaddress(): 42} - raw_tx1 = self.nodes[0].createrawtransaction(tx1_inputs, tx1_outputs) - funded_tx1 = self.nodes[0].fundrawtransaction(raw_tx1) - signed_tx1 = self.nodes[0].signrawtransactionwithwallet(funded_tx1['hex']) - tx1_txid = self.nodes[0].sendrawtransaction(signed_tx1['hex']) + tx1_txid, tx1_vout = self.wallet.send_to( + from_node=node, + scriptPubKey=self.wallet.get_scriptPubKey(), + amount=21 * COIN, + ) # Find the right position of the 21 BTC output - tx1_final = self.nodes[0].gettransaction(tx1_txid) - for output in tx1_final['details']: - if output['amount'] == Decimal('21.00000000') and output['category'] == 'receive': - n = output['vout'] + tx1_out_21 = self.wallet.get_utxo(txid=tx1_txid, vout=tx1_vout) # Generate and send another tx with an OP_RETURN output (which is unspendable) - tx2 = CTransaction() - tx2.vin.append(CTxIn(COutPoint(int(tx1_txid, 16), n), b'')) - tx2.vout.append(CTxOut(int(Decimal('20.99') * COIN), CScript([OP_RETURN] + [OP_FALSE]*30))) - tx2_hex = self.nodes[0].signrawtransactionwithwallet(tx2.serialize().hex())['hex'] + tx2 = self.wallet.create_self_transfer(utxo_to_spend=tx1_out_21)['tx'] + tx2.vout = [CTxOut(int(Decimal('20.99') * COIN), CScript([OP_RETURN] + [OP_FALSE] * 30))] + tx2_hex = tx2.serialize().hex() self.nodes[0].sendrawtransaction(tx2_hex) # Include both txs in a block @@ -177,14 +169,14 @@ class CoinStatsIndexTest(BitcoinTestFramework): assert_equal(res6['total_unspendable_amount'], Decimal('70.99000000')) assert_equal(res6['block_info'], { 'unspendable': Decimal('20.99000000'), - 'prevout_spent': 111, - 'new_outputs_ex_coinbase': Decimal('89.99993620'), - 'coinbase': Decimal('50.01006380'), + 'prevout_spent': 71, + 'new_outputs_ex_coinbase': Decimal('49.99999000'), + 'coinbase': Decimal('50.01001000'), 'unspendables': { 'genesis_block': 0, 'bip30': 0, 'scripts': Decimal('20.99000000'), - 'unclaimed_rewards': 0 + 'unclaimed_rewards': 0, } }) self.block_sanity_check(res6['block_info']) @@ -246,7 +238,7 @@ class CoinStatsIndexTest(BitcoinTestFramework): # Generate two block, let the index catch up, then invalidate the blocks index_node = self.nodes[1] - reorg_blocks = self.generatetoaddress(index_node, 2, index_node.getnewaddress()) + reorg_blocks = self.generatetoaddress(index_node, 2, getnewdestination()[2]) reorg_block = reorg_blocks[1] res_invalid = index_node.gettxoutsetinfo('muhash') index_node.invalidateblock(reorg_blocks[0]) diff --git a/test/functional/feature_maxuploadtarget.py b/test/functional/feature_maxuploadtarget.py index 24f79dda67..0b9d651226 100755 --- a/test/functional/feature_maxuploadtarget.py +++ b/test/functional/feature_maxuploadtarget.py @@ -13,10 +13,19 @@ if uploadtarget has been reached. from collections import defaultdict import time -from test_framework.messages import CInv, MSG_BLOCK, msg_getdata +from test_framework.messages import ( + CInv, + MSG_BLOCK, + msg_getdata, +) from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, mine_large_block +from test_framework.util import ( + assert_equal, + mine_large_block, +) +from test_framework.wallet import MiniWallet + class TestP2PConn(P2PInterface): def __init__(self): @@ -41,12 +50,6 @@ class MaxUploadTest(BitcoinTestFramework): ]] self.supports_cli = False - # Cache for utxos, as the listunspent may take a long time later in the test - self.utxo_cache = [] - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def run_test(self): # Before we connect anything, we first set the time on the node # to be in the past, otherwise things break because the CNode @@ -55,7 +58,8 @@ class MaxUploadTest(BitcoinTestFramework): self.nodes[0].setmocktime(old_time) # Generate some old blocks - self.generate(self.nodes[0], 130) + self.wallet = MiniWallet(self.nodes[0]) + self.generate(self.wallet, 130) # p2p_conns[0] will only request old blocks # p2p_conns[1] will only request new blocks @@ -66,7 +70,7 @@ class MaxUploadTest(BitcoinTestFramework): p2p_conns.append(self.nodes[0].add_p2p_connection(TestP2PConn())) # Now mine a big block - mine_large_block(self, self.nodes[0], self.utxo_cache) + mine_large_block(self, self.wallet, self.nodes[0]) # Store the hash; we'll request this later big_old_block = self.nodes[0].getbestblockhash() @@ -77,7 +81,7 @@ class MaxUploadTest(BitcoinTestFramework): self.nodes[0].setmocktime(int(time.time()) - 2*60*60*24) # Mine one more block, so that the prior block looks old - mine_large_block(self, self.nodes[0], self.utxo_cache) + mine_large_block(self, self.wallet, self.nodes[0]) # We'll be requesting this new block too big_new_block = self.nodes[0].getbestblockhash() diff --git a/test/functional/feature_proxy.py b/test/functional/feature_proxy.py index fb0f6d7cb7..8541c3ed88 100755 --- a/test/functional/feature_proxy.py +++ b/test/functional/feature_proxy.py @@ -30,6 +30,12 @@ addnode connect to generic DNS name addnode connect to a CJDNS address - Test getnetworkinfo for each node + +- Test passing invalid -proxy +- Test passing invalid -onion +- Test passing invalid -i2psam +- Test passing -onlynet=onion without -proxy or -onion +- Test passing -onlynet=onion with -onion=0 and with -noonion """ import socket @@ -234,7 +240,15 @@ class ProxyTest(BitcoinTestFramework): return r self.log.info("Test RPC getnetworkinfo") - n0 = networks_dict(self.nodes[0].getnetworkinfo()) + nodes_network_info = [] + + self.log.debug("Test that setting -proxy disables local address discovery, i.e. -discover=0") + for node in self.nodes: + network_info = node.getnetworkinfo() + assert_equal(network_info["localaddresses"], []) + nodes_network_info.append(network_info) + + n0 = networks_dict(nodes_network_info[0]) assert_equal(NETWORKS, n0.keys()) for net in NETWORKS: if net == NET_I2P: @@ -249,7 +263,7 @@ class ProxyTest(BitcoinTestFramework): assert_equal(n0['i2p']['reachable'], False) assert_equal(n0['cjdns']['reachable'], False) - n1 = networks_dict(self.nodes[1].getnetworkinfo()) + n1 = networks_dict(nodes_network_info[1]) assert_equal(NETWORKS, n1.keys()) for net in ['ipv4', 'ipv6']: assert_equal(n1[net]['proxy'], f'{self.conf1.addr[0]}:{self.conf1.addr[1]}') @@ -261,14 +275,15 @@ class ProxyTest(BitcoinTestFramework): assert_equal(n1['i2p']['proxy_randomize_credentials'], False) assert_equal(n1['i2p']['reachable'], True) - n2 = networks_dict(self.nodes[2].getnetworkinfo()) + n2 = networks_dict(nodes_network_info[2]) assert_equal(NETWORKS, n2.keys()) + proxy = f'{self.conf2.addr[0]}:{self.conf2.addr[1]}' for net in NETWORKS: if net == NET_I2P: expected_proxy = '' expected_randomize = False else: - expected_proxy = f'{self.conf2.addr[0]}:{self.conf2.addr[1]}' + expected_proxy = proxy expected_randomize = True assert_equal(n2[net]['proxy'], expected_proxy) assert_equal(n2[net]['proxy_randomize_credentials'], expected_randomize) @@ -277,20 +292,18 @@ class ProxyTest(BitcoinTestFramework): assert_equal(n2['cjdns']['reachable'], False) if self.have_ipv6: - n3 = networks_dict(self.nodes[3].getnetworkinfo()) + n3 = networks_dict(nodes_network_info[3]) assert_equal(NETWORKS, n3.keys()) + proxy = f'[{self.conf3.addr[0]}]:{self.conf3.addr[1]}' for net in NETWORKS: - if net == NET_I2P or net == NET_ONION: - expected_proxy = '' - else: - expected_proxy = f'[{self.conf3.addr[0]}]:{self.conf3.addr[1]}' + expected_proxy = '' if net == NET_I2P or net == NET_ONION else proxy assert_equal(n3[net]['proxy'], expected_proxy) assert_equal(n3[net]['proxy_randomize_credentials'], False) assert_equal(n3['onion']['reachable'], False) assert_equal(n3['i2p']['reachable'], False) assert_equal(n3['cjdns']['reachable'], False) - n4 = networks_dict(self.nodes[4].getnetworkinfo()) + n4 = networks_dict(nodes_network_info[4]) assert_equal(NETWORKS, n4.keys()) for net in NETWORKS: if net == NET_I2P: @@ -305,6 +318,37 @@ class ProxyTest(BitcoinTestFramework): assert_equal(n4['i2p']['reachable'], False) assert_equal(n4['cjdns']['reachable'], True) + self.stop_node(1) + + self.log.info("Test passing invalid -proxy raises expected init error") + self.nodes[1].extra_args = ["-proxy=abc:def"] + msg = "Error: Invalid -proxy address or hostname: 'abc:def'" + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + + self.log.info("Test passing invalid -onion raises expected init error") + self.nodes[1].extra_args = ["-onion=xyz:abc"] + msg = "Error: Invalid -onion address or hostname: 'xyz:abc'" + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + + self.log.info("Test passing invalid -i2psam raises expected init error") + self.nodes[1].extra_args = ["-i2psam=def:xyz"] + msg = "Error: Invalid -i2psam address or hostname: 'def:xyz'" + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + + msg = ( + "Error: Outbound connections restricted to Tor (-onlynet=onion) but " + "the proxy for reaching the Tor network is not provided (no -proxy= " + "and no -onion= given) or it is explicitly forbidden (-onion=0)" + ) + self.log.info("Test passing -onlynet=onion without -proxy or -onion raises expected init error") + self.nodes[1].extra_args = ["-onlynet=onion"] + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + + self.log.info("Test passing -onlynet=onion with -onion=0/-noonion raises expected init error") + for arg in ["-onion=0", "-noonion"]: + self.nodes[1].extra_args = ["-onlynet=onion", arg] + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + if __name__ == '__main__': ProxyTest().main() diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index ba3c5053cb..bf19384279 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -141,6 +141,10 @@ class PruneTest(BitcoinTestFramework): expected_msg='Error: Prune mode is incompatible with -coinstatsindex.', extra_args=['-prune=550', '-coinstatsindex'], ) + self.nodes[0].assert_start_raises_init_error( + expected_msg='Error: Prune mode is incompatible with -reindex-chainstate. Use full -reindex instead.', + extra_args=['-prune=550', '-reindex-chainstate'], + ) def test_height_min(self): assert os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), "blk00000.dat is missing, pruning too early" diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 6d7f1def88..f0faf1421b 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -86,18 +86,18 @@ class SegWitTest(BitcoinTestFramework): [ "-acceptnonstdtxn=1", "-rpcserialversion=0", - "-testactivationheight=segwit@432", + "-testactivationheight=segwit@165", "-addresstype=legacy", ], [ "-acceptnonstdtxn=1", "-rpcserialversion=1", - "-testactivationheight=segwit@432", + "-testactivationheight=segwit@165", "-addresstype=legacy", ], [ "-acceptnonstdtxn=1", - "-testactivationheight=segwit@432", + "-testactivationheight=segwit@165", "-addresstype=legacy", ], ] @@ -117,12 +117,6 @@ class SegWitTest(BitcoinTestFramework): assert_equal(len(node.getblock(block[0])["tx"]), 2) self.sync_blocks() - def skip_mine(self, node, txid, sign, redeem_script=""): - send_to_witness(1, node, getutxo(txid), self.pubkey[0], False, Decimal("49.998"), sign, redeem_script) - block = self.generate(node, 1) - assert_equal(len(node.getblock(block[0])["tx"]), 1) - self.sync_blocks() - def fail_accept(self, node, error_msg, txid, sign, redeem_script=""): assert_raises_rpc_error(-26, error_msg, send_to_witness, use_p2wsh=1, node=node, utxo=getutxo(txid), pubkey=self.pubkey[0], encode_p2sh=False, amount=Decimal("49.998"), sign=sign, insert_redeem_script=redeem_script) @@ -197,23 +191,21 @@ class SegWitTest(BitcoinTestFramework): assert_equal(self.nodes[1].getbalance(), 20 * Decimal("49.999")) assert_equal(self.nodes[2].getbalance(), 20 * Decimal("49.999")) - self.generate(self.nodes[0], 260) # block 423 - - self.log.info("Verify witness txs are skipped for mining before the fork") - self.skip_mine(self.nodes[2], wit_ids[NODE_2][P2WPKH][0], True) # block 424 - self.skip_mine(self.nodes[2], wit_ids[NODE_2][P2WSH][0], True) # block 425 - self.skip_mine(self.nodes[2], p2sh_ids[NODE_2][P2WPKH][0], True) # block 426 - self.skip_mine(self.nodes[2], p2sh_ids[NODE_2][P2WSH][0], True) # block 427 - self.log.info("Verify unsigned p2sh witness txs without a redeem script are invalid") self.fail_accept(self.nodes[2], "mandatory-script-verify-flag-failed (Operation not valid with the current stack size)", p2sh_ids[NODE_2][P2WPKH][1], sign=False) self.fail_accept(self.nodes[2], "mandatory-script-verify-flag-failed (Operation not valid with the current stack size)", p2sh_ids[NODE_2][P2WSH][1], sign=False) - self.generate(self.nodes[2], 4) # blocks 428-431 + self.generate(self.nodes[0], 1) # block 164 + + self.log.info("Verify witness txs are mined as soon as segwit activates") + + send_to_witness(1, self.nodes[2], getutxo(wit_ids[NODE_2][P2WPKH][0]), self.pubkey[0], encode_p2sh=False, amount=Decimal("49.998"), sign=True) + send_to_witness(1, self.nodes[2], getutxo(wit_ids[NODE_2][P2WSH][0]), self.pubkey[0], encode_p2sh=False, amount=Decimal("49.998"), sign=True) + send_to_witness(1, self.nodes[2], getutxo(p2sh_ids[NODE_2][P2WPKH][0]), self.pubkey[0], encode_p2sh=False, amount=Decimal("49.998"), sign=True) + send_to_witness(1, self.nodes[2], getutxo(p2sh_ids[NODE_2][P2WSH][0]), self.pubkey[0], encode_p2sh=False, amount=Decimal("49.998"), sign=True) - self.log.info("Verify previous witness txs skipped for mining can now be mined") assert_equal(len(self.nodes[2].getrawmempool()), 4) - blockhash = self.generate(self.nodes[2], 1)[0] # block 432 (first block with new rules; 432 = 144 * 3) + blockhash = self.generate(self.nodes[2], 1)[0] # block 165 (first block with new rules) assert_equal(len(self.nodes[2].getrawmempool()), 0) segwit_tx_list = self.nodes[2].getblock(blockhash)["tx"] assert_equal(len(segwit_tx_list), 5) @@ -255,10 +247,10 @@ class SegWitTest(BitcoinTestFramework): self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program was passed an empty witness)', p2sh_ids[NODE_2][P2WSH][2], sign=False, redeem_script=witness_script(True, self.pubkey[2])) self.log.info("Verify default node can now use witness txs") - self.success_mine(self.nodes[0], wit_ids[NODE_0][P2WPKH][0], True) # block 432 - self.success_mine(self.nodes[0], wit_ids[NODE_0][P2WSH][0], True) # block 433 - self.success_mine(self.nodes[0], p2sh_ids[NODE_0][P2WPKH][0], True) # block 434 - self.success_mine(self.nodes[0], p2sh_ids[NODE_0][P2WSH][0], True) # block 435 + self.success_mine(self.nodes[0], wit_ids[NODE_0][P2WPKH][0], True) + self.success_mine(self.nodes[0], wit_ids[NODE_0][P2WSH][0], True) + self.success_mine(self.nodes[0], p2sh_ids[NODE_0][P2WPKH][0], True) + self.success_mine(self.nodes[0], p2sh_ids[NODE_0][P2WSH][0], True) self.log.info("Verify sigops are counted in GBT with BIP141 rules after the fork") txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index 3e3d4b3c77..c3925dbb00 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -1182,15 +1182,11 @@ def spenders_taproot_inactive(): ] tap = taproot_construct(pub, scripts) - # Test that keypath spending is valid & non-standard, regardless of validity. + # Test that valid spending is standard. add_spender(spenders, "inactive/keypath_valid", key=sec, tap=tap, standard=Standard.V23) - add_spender(spenders, "inactive/keypath_invalidsig", key=sec, tap=tap, standard=False, sighash=bitflipper(default_sighash)) - add_spender(spenders, "inactive/keypath_empty", key=sec, tap=tap, standard=False, witness=[]) - - # Same for scriptpath spending (and features like annex, leaf versions, or OP_SUCCESS don't change this) add_spender(spenders, "inactive/scriptpath_valid", key=sec, tap=tap, leaf="pk", standard=Standard.V23, inputs=[getter("sign")]) - add_spender(spenders, "inactive/scriptpath_invalidsig", key=sec, tap=tap, leaf="pk", standard=False, inputs=[getter("sign")], sighash=bitflipper(default_sighash)) - add_spender(spenders, "inactive/scriptpath_invalidcb", key=sec, tap=tap, leaf="pk", standard=False, inputs=[getter("sign")], controlblock=bitflipper(default_controlblock)) + + # Test that features like annex, leaf versions, or OP_SUCCESS are valid but non-standard add_spender(spenders, "inactive/scriptpath_valid_unkleaf", key=sec, tap=tap, leaf="future_leaf", standard=False, inputs=[getter("sign")]) add_spender(spenders, "inactive/scriptpath_invalid_unkleaf", key=sec, tap=tap, leaf="future_leaf", standard=False, inputs=[getter("sign")], sighash=bitflipper(default_sighash)) add_spender(spenders, "inactive/scriptpath_valid_opsuccess", key=sec, tap=tap, leaf="op_success", standard=False, inputs=[getter("sign")]) diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 1ee12c0040..7d8d10589b 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -23,6 +23,9 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, ) +from test_framework.wallet import ( + MiniWallet, +) from test_framework.netutil import test_ipv6_local from io import BytesIO from time import sleep @@ -100,8 +103,6 @@ class ZMQTestSetupBlock: class ZMQTest (BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - if self.is_wallet_compiled(): - self.requires_wallet = True # This test isn't testing txn relay/timing, so set whitelist on the # peers for instant txn relay. This speeds up the test run time 2-3x. self.extra_args = [["-whitelist=noban@127.0.0.1"]] * self.num_nodes @@ -111,6 +112,7 @@ class ZMQTest (BitcoinTestFramework): self.skip_if_no_bitcoind_zmq() def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) self.ctx = zmq.Context() try: self.test_basic() @@ -211,25 +213,25 @@ class ZMQTest (BitcoinTestFramework): assert_equal([txid.hex()], self.nodes[1].getblock(hash)["tx"]) - if self.is_wallet_compiled(): - self.log.info("Wait for tx from second node") - payment_txid = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1.0) - self.sync_all() - - # Should receive the broadcasted txid. - txid = hashtx.receive() - assert_equal(payment_txid, txid.hex()) + self.wallet.rescan_utxos() + self.log.info("Wait for tx from second node") + payment_tx = self.wallet.send_self_transfer(from_node=self.nodes[1]) + payment_txid = payment_tx['txid'] + self.sync_all() + # Should receive the broadcasted txid. + txid = hashtx.receive() + assert_equal(payment_txid, txid.hex()) - # Should receive the broadcasted raw transaction. - hex = rawtx.receive() - assert_equal(payment_txid, hash256_reversed(hex).hex()) + # Should receive the broadcasted raw transaction. + hex = rawtx.receive() + assert_equal(payment_tx['wtxid'], hash256_reversed(hex).hex()) - # Mining the block with this tx should result in second notification - # after coinbase tx notification - self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) - hashtx.receive() - txid = hashtx.receive() - assert_equal(payment_txid, txid.hex()) + # Mining the block with this tx should result in second notification + # after coinbase tx notification + self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) + hashtx.receive() + txid = hashtx.receive() + assert_equal(payment_txid, txid.hex()) self.log.info("Test the getzmqnotifications RPC") @@ -243,9 +245,6 @@ class ZMQTest (BitcoinTestFramework): assert_equal(self.nodes[1].getzmqnotifications(), []) def test_reorg(self): - if not self.is_wallet_compiled(): - self.log.info("Skipping reorg test because wallet is disabled") - return address = 'tcp://127.0.0.1:28333' @@ -256,7 +255,7 @@ class ZMQTest (BitcoinTestFramework): self.disconnect_nodes(0, 1) # Generate 1 block in nodes[0] with 1 mempool tx and receive all notifications - payment_txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1.0) + payment_txid = self.wallet.send_self_transfer(from_node=self.nodes[0])['txid'] disconnect_block = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE, sync_fun=self.no_op)[0] disconnect_cb = self.nodes[0].getblock(disconnect_block)["tx"][0] assert_equal(self.nodes[0].getbestblockhash(), hashblock.receive().hex()) @@ -325,126 +324,124 @@ class ZMQTest (BitcoinTestFramework): assert_equal((self.nodes[1].getblockhash(block_count-1), "C", None), seq.receive_sequence()) assert_equal((self.nodes[1].getblockhash(block_count), "C", None), seq.receive_sequence()) - # Rest of test requires wallet functionality - if self.is_wallet_compiled(): - self.log.info("Wait for tx from second node") - payment_txid = self.nodes[1].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=5.0, replaceable=True) - self.sync_all() - self.log.info("Testing sequence notifications with mempool sequence values") - - # Should receive the broadcasted txid. - assert_equal((payment_txid, "A", seq_num), seq.receive_sequence()) - seq_num += 1 - - self.log.info("Testing RBF notification") - # Replace it to test eviction/addition notification - rbf_info = self.nodes[1].bumpfee(payment_txid) - self.sync_all() - assert_equal((payment_txid, "R", seq_num), seq.receive_sequence()) - seq_num += 1 - assert_equal((rbf_info["txid"], "A", seq_num), seq.receive_sequence()) - seq_num += 1 - - # Doesn't get published when mined, make a block and tx to "flush" the possibility - # though the mempool sequence number does go up by the number of transactions - # removed from the mempool by the block mining it. - mempool_size = len(self.nodes[0].getrawmempool()) - c_block = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE)[0] - # Make sure the number of mined transactions matches the number of txs out of mempool - mempool_size_delta = mempool_size - len(self.nodes[0].getrawmempool()) - assert_equal(len(self.nodes[0].getblock(c_block)["tx"])-1, mempool_size_delta) - seq_num += mempool_size_delta - payment_txid_2 = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1.0) - self.sync_all() - assert_equal((c_block, "C", None), seq.receive_sequence()) - assert_equal((payment_txid_2, "A", seq_num), seq.receive_sequence()) - seq_num += 1 - - # Spot check getrawmempool results that they only show up when asked for - assert type(self.nodes[0].getrawmempool()) is list - assert type(self.nodes[0].getrawmempool(mempool_sequence=False)) is list - assert "mempool_sequence" not in self.nodes[0].getrawmempool(verbose=True) - assert_raises_rpc_error(-8, "Verbose results cannot contain mempool sequence values.", self.nodes[0].getrawmempool, True, True) - assert_equal(self.nodes[0].getrawmempool(mempool_sequence=True)["mempool_sequence"], seq_num) - - self.log.info("Testing reorg notifications") - # Manually invalidate the last block to test mempool re-entry - # N.B. This part could be made more lenient in exact ordering - # since it greatly depends on inner-workings of blocks/mempool - # during "deep" re-orgs. Probably should "re-construct" - # blockchain/mempool state from notifications instead. - block_count = self.nodes[0].getblockcount() - best_hash = self.nodes[0].getbestblockhash() - self.nodes[0].invalidateblock(best_hash) - sleep(2) # Bit of room to make sure transaction things happened - - # Make sure getrawmempool mempool_sequence results aren't "queued" but immediately reflective - # of the time they were gathered. - assert self.nodes[0].getrawmempool(mempool_sequence=True)["mempool_sequence"] > seq_num - - assert_equal((best_hash, "D", None), seq.receive_sequence()) - assert_equal((rbf_info["txid"], "A", seq_num), seq.receive_sequence()) - seq_num += 1 - - # Other things may happen but aren't wallet-deterministic so we don't test for them currently - self.nodes[0].reconsiderblock(best_hash) - self.generatetoaddress(self.nodes[1], 1, ADDRESS_BCRT1_UNSPENDABLE) - - self.log.info("Evict mempool transaction by block conflict") - orig_txid = self.nodes[0].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=1.0, replaceable=True) - - # More to be simply mined - more_tx = [] - for _ in range(5): - more_tx.append(self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 0.1)) - - raw_tx = self.nodes[0].getrawtransaction(orig_txid) - bump_info = self.nodes[0].bumpfee(orig_txid) - # Mine the pre-bump tx - txs_to_add = [raw_tx] + [self.nodes[0].getrawtransaction(txid) for txid in more_tx] - block = create_block(int(self.nodes[0].getbestblockhash(), 16), create_coinbase(self.nodes[0].getblockcount()+1), txlist=txs_to_add) - add_witness_commitment(block) - block.solve() - assert_equal(self.nodes[0].submitblock(block.serialize().hex()), None) - tip = self.nodes[0].getbestblockhash() - assert_equal(int(tip, 16), block.sha256) - orig_txid_2 = self.nodes[0].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=1.0, replaceable=True) - - # Flush old notifications until evicted tx original entry + self.log.info("Wait for tx from second node") + payment_tx = self.wallet.send_self_transfer(from_node=self.nodes[1]) + payment_txid = payment_tx['txid'] + self.sync_all() + self.log.info("Testing sequence notifications with mempool sequence values") + + # Should receive the broadcasted txid. + assert_equal((payment_txid, "A", seq_num), seq.receive_sequence()) + seq_num += 1 + + self.log.info("Testing RBF notification") + # Replace it to test eviction/addition notification + payment_tx['tx'].vout[0].nValue -= 1000 + rbf_txid = self.nodes[1].sendrawtransaction(payment_tx['tx'].serialize().hex()) + self.sync_all() + assert_equal((payment_txid, "R", seq_num), seq.receive_sequence()) + seq_num += 1 + assert_equal((rbf_txid, "A", seq_num), seq.receive_sequence()) + seq_num += 1 + + # Doesn't get published when mined, make a block and tx to "flush" the possibility + # though the mempool sequence number does go up by the number of transactions + # removed from the mempool by the block mining it. + mempool_size = len(self.nodes[0].getrawmempool()) + c_block = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE)[0] + # Make sure the number of mined transactions matches the number of txs out of mempool + mempool_size_delta = mempool_size - len(self.nodes[0].getrawmempool()) + assert_equal(len(self.nodes[0].getblock(c_block)["tx"])-1, mempool_size_delta) + seq_num += mempool_size_delta + payment_txid_2 = self.wallet.send_self_transfer(from_node=self.nodes[1])['txid'] + self.sync_all() + assert_equal((c_block, "C", None), seq.receive_sequence()) + assert_equal((payment_txid_2, "A", seq_num), seq.receive_sequence()) + seq_num += 1 + + # Spot check getrawmempool results that they only show up when asked for + assert type(self.nodes[0].getrawmempool()) is list + assert type(self.nodes[0].getrawmempool(mempool_sequence=False)) is list + assert "mempool_sequence" not in self.nodes[0].getrawmempool(verbose=True) + assert_raises_rpc_error(-8, "Verbose results cannot contain mempool sequence values.", self.nodes[0].getrawmempool, True, True) + assert_equal(self.nodes[0].getrawmempool(mempool_sequence=True)["mempool_sequence"], seq_num) + + self.log.info("Testing reorg notifications") + # Manually invalidate the last block to test mempool re-entry + # N.B. This part could be made more lenient in exact ordering + # since it greatly depends on inner-workings of blocks/mempool + # during "deep" re-orgs. Probably should "re-construct" + # blockchain/mempool state from notifications instead. + block_count = self.nodes[0].getblockcount() + best_hash = self.nodes[0].getbestblockhash() + self.nodes[0].invalidateblock(best_hash) + sleep(2) # Bit of room to make sure transaction things happened + + # Make sure getrawmempool mempool_sequence results aren't "queued" but immediately reflective + # of the time they were gathered. + assert self.nodes[0].getrawmempool(mempool_sequence=True)["mempool_sequence"] > seq_num + + assert_equal((best_hash, "D", None), seq.receive_sequence()) + assert_equal((rbf_txid, "A", seq_num), seq.receive_sequence()) + seq_num += 1 + + # Other things may happen but aren't wallet-deterministic so we don't test for them currently + self.nodes[0].reconsiderblock(best_hash) + self.generatetoaddress(self.nodes[1], 1, ADDRESS_BCRT1_UNSPENDABLE) + + self.log.info("Evict mempool transaction by block conflict") + orig_tx = self.wallet.send_self_transfer(from_node=self.nodes[0]) + orig_txid = orig_tx['txid'] + + # More to be simply mined + more_tx = [] + for _ in range(5): + more_tx.append(self.wallet.send_self_transfer(from_node=self.nodes[0])) + + orig_tx['tx'].vout[0].nValue -= 1000 + bump_txid = self.nodes[0].sendrawtransaction(orig_tx['tx'].serialize().hex()) + # Mine the pre-bump tx + txs_to_add = [orig_tx['hex']] + [tx['hex'] for tx in more_tx] + block = create_block(int(self.nodes[0].getbestblockhash(), 16), create_coinbase(self.nodes[0].getblockcount()+1), txlist=txs_to_add) + add_witness_commitment(block) + block.solve() + assert_equal(self.nodes[0].submitblock(block.serialize().hex()), None) + tip = self.nodes[0].getbestblockhash() + assert_equal(int(tip, 16), block.sha256) + orig_txid_2 = self.wallet.send_self_transfer(from_node=self.nodes[0])['txid'] + + # Flush old notifications until evicted tx original entry + (hash_str, label, mempool_seq) = seq.receive_sequence() + while hash_str != orig_txid: (hash_str, label, mempool_seq) = seq.receive_sequence() - while hash_str != orig_txid: - (hash_str, label, mempool_seq) = seq.receive_sequence() - mempool_seq += 1 + mempool_seq += 1 - # Added original tx - assert_equal(label, "A") - # More transactions to be simply mined - for i in range(len(more_tx)): - assert_equal((more_tx[i], "A", mempool_seq), seq.receive_sequence()) - mempool_seq += 1 - # Bumped by rbf - assert_equal((orig_txid, "R", mempool_seq), seq.receive_sequence()) - mempool_seq += 1 - assert_equal((bump_info["txid"], "A", mempool_seq), seq.receive_sequence()) + # Added original tx + assert_equal(label, "A") + # More transactions to be simply mined + for i in range(len(more_tx)): + assert_equal((more_tx[i]['txid'], "A", mempool_seq), seq.receive_sequence()) mempool_seq += 1 - # Conflict announced first, then block - assert_equal((bump_info["txid"], "R", mempool_seq), seq.receive_sequence()) - mempool_seq += 1 - assert_equal((tip, "C", None), seq.receive_sequence()) - mempool_seq += len(more_tx) - # Last tx - assert_equal((orig_txid_2, "A", mempool_seq), seq.receive_sequence()) - mempool_seq += 1 - self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) - self.sync_all() # want to make sure we didn't break "consensus" for other tests + # Bumped by rbf + assert_equal((orig_txid, "R", mempool_seq), seq.receive_sequence()) + mempool_seq += 1 + assert_equal((bump_txid, "A", mempool_seq), seq.receive_sequence()) + mempool_seq += 1 + # Conflict announced first, then block + assert_equal((bump_txid, "R", mempool_seq), seq.receive_sequence()) + mempool_seq += 1 + assert_equal((tip, "C", None), seq.receive_sequence()) + mempool_seq += len(more_tx) + # Last tx + assert_equal((orig_txid_2, "A", mempool_seq), seq.receive_sequence()) + mempool_seq += 1 + self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) + self.sync_all() # want to make sure we didn't break "consensus" for other tests def test_mempool_sync(self): """ Use sequence notification plus getrawmempool sequence results to "sync mempool" """ - if not self.is_wallet_compiled(): - self.log.info("Skipping mempool sync test") - return self.log.info("Testing 'mempool sync' usage of sequence notifier") [seq] = self.setup_zmq_test([("sequence", "tcp://127.0.0.1:28333")]) @@ -455,10 +452,10 @@ class ZMQTest (BitcoinTestFramework): # Some transactions have been happening but we aren't consuming zmq notifications yet # or we lost a ZMQ message somehow and want to start over - txids = [] + txs = [] num_txs = 5 for _ in range(num_txs): - txids.append(self.nodes[1].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=1.0, replaceable=True)) + txs.append(self.wallet.send_self_transfer(from_node=self.nodes[1])) self.sync_all() # 1) Consume backlog until we get a mempool sequence number @@ -484,11 +481,12 @@ class ZMQTest (BitcoinTestFramework): # Things continue to happen in the "interim" while waiting for snapshot results # We have node 0 do all these to avoid p2p races with RBF announcements for _ in range(num_txs): - txids.append(self.nodes[0].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=0.1, replaceable=True)) - self.nodes[0].bumpfee(txids[-1]) + txs.append(self.wallet.send_self_transfer(from_node=self.nodes[0])) + txs[-1]['tx'].vout[0].nValue -= 1000 + self.nodes[0].sendrawtransaction(txs[-1]['tx'].serialize().hex()) self.sync_all() self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) - final_txid = self.nodes[0].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=0.1, replaceable=True) + final_txid = self.wallet.send_self_transfer(from_node=self.nodes[0])['txid'] # 3) Consume ZMQ backlog until we get to "now" for the mempool snapshot while True: diff --git a/test/functional/mempool_package_onemore.py b/test/functional/mempool_package_onemore.py index a6fb1dcf35..423a5bf2ee 100755 --- a/test/functional/mempool_package_onemore.py +++ b/test/functional/mempool_package_onemore.py @@ -7,74 +7,68 @@ size. """ -from decimal import Decimal - -from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, - chain_transaction, ) +from test_framework.wallet import MiniWallet + MAX_ANCESTORS = 25 MAX_DESCENDANTS = 25 + class MempoolPackagesTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.extra_args = [["-maxorphantx=1000"]] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + def chain_tx(self, utxos_to_spend, *, num_outputs=1): + return self.wallet.send_self_transfer_multi( + from_node=self.nodes[0], + utxos_to_spend=utxos_to_spend, + num_outputs=num_outputs)['new_utxos'] def run_test(self): - # Mine some blocks and have them mature. - self.generate(self.nodes[0], COINBASE_MATURITY + 1) - utxo = self.nodes[0].listunspent(10) - txid = utxo[0]['txid'] - vout = utxo[0]['vout'] - value = utxo[0]['amount'] + self.wallet = MiniWallet(self.nodes[0]) + self.wallet.rescan_utxos() - fee = Decimal("0.0002") # MAX_ANCESTORS transactions off a confirmed tx should be fine chain = [] + utxo = self.wallet.get_utxo() for _ in range(4): - (txid, sent_value) = chain_transaction(self.nodes[0], [txid], [vout], value, fee, 2) - vout = 0 - value = sent_value - chain.append([txid, value]) + utxo, utxo2 = self.chain_tx([utxo], num_outputs=2) + chain.append(utxo2) for _ in range(MAX_ANCESTORS - 4): - (txid, sent_value) = chain_transaction(self.nodes[0], [txid], [0], value, fee, 1) - value = sent_value - chain.append([txid, value]) - (second_chain, second_chain_value) = chain_transaction(self.nodes[0], [utxo[1]['txid']], [utxo[1]['vout']], utxo[1]['amount'], fee, 1) + utxo, = self.chain_tx([utxo]) + chain.append(utxo) + second_chain, = self.chain_tx([self.wallet.get_utxo()]) # Check mempool has MAX_ANCESTORS + 1 transactions in it assert_equal(len(self.nodes[0].getrawmempool()), MAX_ANCESTORS + 1) # Adding one more transaction on to the chain should fail. - assert_raises_rpc_error(-26, "too-long-mempool-chain, too many unconfirmed ancestors [limit: 25]", chain_transaction, self.nodes[0], [txid], [0], value, fee, 1) + assert_raises_rpc_error(-26, "too-long-mempool-chain, too many unconfirmed ancestors [limit: 25]", self.chain_tx, [utxo]) # ...even if it chains on from some point in the middle of the chain. - assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", chain_transaction, self.nodes[0], [chain[2][0]], [1], chain[2][1], fee, 1) - assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", chain_transaction, self.nodes[0], [chain[1][0]], [1], chain[1][1], fee, 1) + assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[2]]) + assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[1]]) # ...even if it chains on to two parent transactions with one in the chain. - assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", chain_transaction, self.nodes[0], [chain[0][0], second_chain], [1, 0], chain[0][1] + second_chain_value, fee, 1) + assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[0], second_chain]) # ...especially if its > 40k weight - assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", chain_transaction, self.nodes[0], [chain[0][0]], [1], chain[0][1], fee, 350) + assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[0]], num_outputs=350) # But not if it chains directly off the first transaction - (replacable_txid, replacable_orig_value) = chain_transaction(self.nodes[0], [chain[0][0]], [1], chain[0][1], fee, 1) + replacable_tx = self.wallet.send_self_transfer_multi(from_node=self.nodes[0], utxos_to_spend=[chain[0]])['tx'] # and the second chain should work just fine - chain_transaction(self.nodes[0], [second_chain], [0], second_chain_value, fee, 1) + self.chain_tx([second_chain]) # Make sure we can RBF the chain which used our carve-out rule - second_tx_outputs = {self.nodes[0].getrawtransaction(replacable_txid, True)["vout"][0]['scriptPubKey']['address']: replacable_orig_value - (Decimal(1) / Decimal(100))} - second_tx = self.nodes[0].createrawtransaction([{'txid': chain[0][0], 'vout': 1}], second_tx_outputs) - signed_second_tx = self.nodes[0].signrawtransactionwithwallet(second_tx) - self.nodes[0].sendrawtransaction(signed_second_tx['hex']) + replacable_tx.vout[0].nValue -= 1000000 + self.nodes[0].sendrawtransaction(replacable_tx.serialize().hex()) # Finally, check that we added two transactions assert_equal(len(self.nodes[0].getrawmempool()), MAX_ANCESTORS + 3) + if __name__ == '__main__': MempoolPackagesTest().main() diff --git a/test/functional/mempool_unbroadcast.py b/test/functional/mempool_unbroadcast.py index adf7326dac..37ef4a9157 100755 --- a/test/functional/mempool_unbroadcast.py +++ b/test/functional/mempool_unbroadcast.py @@ -9,21 +9,20 @@ import time from test_framework.p2p import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_equal, - create_confirmed_utxos, -) +from test_framework.util import assert_equal +from test_framework.wallet import MiniWallet MAX_INITIAL_BROADCAST_DELAY = 15 * 60 # 15 minutes in seconds class MempoolUnbroadcastTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + if self.is_wallet_compiled(): + self.requires_wallet = True def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + self.wallet.rescan_utxos() self.test_broadcast() self.test_txn_removal() @@ -31,30 +30,25 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): self.log.info("Test that mempool reattempts delivery of locally submitted transaction") node = self.nodes[0] - min_relay_fee = node.getnetworkinfo()["relayfee"] - utxos = create_confirmed_utxos(self, min_relay_fee, node, 10) - self.disconnect_nodes(0, 1) self.log.info("Generate transactions that only node 0 knows about") - # generate a wallet txn - addr = node.getnewaddress() - wallet_tx_hsh = node.sendtoaddress(addr, 0.0001) + if self.is_wallet_compiled(): + # generate a wallet txn + addr = node.getnewaddress() + wallet_tx_hsh = node.sendtoaddress(addr, 0.0001) # generate a txn using sendrawtransaction - us0 = utxos.pop() - inputs = [{"txid": us0["txid"], "vout": us0["vout"]}] - outputs = {addr: 0.0001} - tx = node.createrawtransaction(inputs, outputs) - node.settxfee(min_relay_fee) - txF = node.fundrawtransaction(tx) - txFS = node.signrawtransactionwithwallet(txF["hex"]) + txFS = self.wallet.create_self_transfer(from_node=node) rpc_tx_hsh = node.sendrawtransaction(txFS["hex"]) # check transactions are in unbroadcast using rpc mempoolinfo = self.nodes[0].getmempoolinfo() - assert_equal(mempoolinfo['unbroadcastcount'], 2) + unbroadcast_count = 1 + if self.is_wallet_compiled(): + unbroadcast_count += 1 + assert_equal(mempoolinfo['unbroadcastcount'], unbroadcast_count) mempool = self.nodes[0].getrawmempool(True) for tx in mempool: assert_equal(mempool[tx]['unbroadcast'], True) @@ -62,7 +56,8 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): # check that second node doesn't have these two txns mempool = self.nodes[1].getrawmempool() assert rpc_tx_hsh not in mempool - assert wallet_tx_hsh not in mempool + if self.is_wallet_compiled(): + assert wallet_tx_hsh not in mempool # ensure that unbroadcast txs are persisted to mempool.dat self.restart_node(0) @@ -75,7 +70,8 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): self.sync_mempools(timeout=30) mempool = self.nodes[1].getrawmempool() assert rpc_tx_hsh in mempool - assert wallet_tx_hsh in mempool + if self.is_wallet_compiled(): + assert wallet_tx_hsh in mempool # check that transactions are no longer in first node's unbroadcast set mempool = self.nodes[0].getrawmempool(True) @@ -102,8 +98,7 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): # since the node doesn't have any connections, it will not receive # any GETDATAs & thus the transaction will remain in the unbroadcast set. - addr = node.getnewaddress() - txhsh = node.sendtoaddress(addr, 0.0001) + txhsh = self.wallet.send_self_transfer(from_node=node)["txid"] # check transaction was removed from unbroadcast set due to presence in # a block diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index 6f2ac805a0..a15fbe5a24 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -4,11 +4,15 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the prioritisetransaction mining RPC.""" +from decimal import Decimal import time +from test_framework.blocktools import COINBASE_MATURITY from test_framework.messages import COIN, MAX_BLOCK_WEIGHT from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error, create_confirmed_utxos, create_lots_of_big_transactions, gen_return_txouts +from test_framework.wallet import MiniWallet + class PrioritiseTransactionTest(BitcoinTestFramework): def set_test_params(self): @@ -23,7 +27,84 @@ class PrioritiseTransactionTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() + def test_diamond(self): + self.log.info("Test diamond-shape package with priority") + self.generate(self.wallet, COINBASE_MATURITY + 1) + mock_time = int(time.time()) + self.nodes[0].setmocktime(mock_time) + + # tx_a + # / \ + # / \ + # tx_b tx_c + # \ / + # \ / + # tx_d + + tx_o_a = self.wallet.send_self_transfer_multi( + from_node=self.nodes[0], + num_outputs=2, + ) + txid_a = tx_o_a["txid"] + + tx_o_b, tx_o_c = [self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=u, + ) for u in tx_o_a["new_utxos"]] + txid_b = tx_o_b["txid"] + txid_c = tx_o_c["txid"] + + tx_o_d = self.wallet.send_self_transfer_multi( + from_node=self.nodes[0], + utxos_to_spend=[ + self.wallet.get_utxo(txid=txid_b), + self.wallet.get_utxo(txid=txid_c), + ], + ) + txid_d = tx_o_d["txid"] + + self.log.info("Test priority while txs are in mempool") + raw_before = self.nodes[0].getrawmempool(verbose=True) + fee_delta_b = Decimal(9999) / COIN + fee_delta_c_1 = Decimal(-1234) / COIN + fee_delta_c_2 = Decimal(8888) / COIN + self.nodes[0].prioritisetransaction(txid=txid_b, fee_delta=int(fee_delta_b * COIN)) + self.nodes[0].prioritisetransaction(txid=txid_c, fee_delta=int(fee_delta_c_1 * COIN)) + self.nodes[0].prioritisetransaction(txid=txid_c, fee_delta=int(fee_delta_c_2 * COIN)) + raw_before[txid_a]["fees"]["descendant"] += fee_delta_b + fee_delta_c_1 + fee_delta_c_2 + raw_before[txid_b]["fees"]["modified"] += fee_delta_b + raw_before[txid_b]["fees"]["ancestor"] += fee_delta_b + raw_before[txid_b]["fees"]["descendant"] += fee_delta_b + raw_before[txid_c]["fees"]["modified"] += fee_delta_c_1 + fee_delta_c_2 + raw_before[txid_c]["fees"]["ancestor"] += fee_delta_c_1 + fee_delta_c_2 + raw_before[txid_c]["fees"]["descendant"] += fee_delta_c_1 + fee_delta_c_2 + raw_before[txid_d]["fees"]["ancestor"] += fee_delta_b + fee_delta_c_1 + fee_delta_c_2 + raw_after = self.nodes[0].getrawmempool(verbose=True) + assert_equal(raw_before[txid_a], raw_after[txid_a]) + assert_equal(raw_before, raw_after) + + self.log.info("Test priority while txs are not in mempool") + self.restart_node(0, extra_args=["-nopersistmempool"]) + self.nodes[0].setmocktime(mock_time) + assert_equal(self.nodes[0].getmempoolinfo()["size"], 0) + self.nodes[0].prioritisetransaction(txid=txid_b, fee_delta=int(fee_delta_b * COIN)) + self.nodes[0].prioritisetransaction(txid=txid_c, fee_delta=int(fee_delta_c_1 * COIN)) + self.nodes[0].prioritisetransaction(txid=txid_c, fee_delta=int(fee_delta_c_2 * COIN)) + for t in [tx_o_a["hex"], tx_o_b["hex"], tx_o_c["hex"], tx_o_d["hex"]]: + self.nodes[0].sendrawtransaction(t) + raw_after = self.nodes[0].getrawmempool(verbose=True) + assert_equal(raw_before[txid_a], raw_after[txid_a]) + assert_equal(raw_before, raw_after) + + # Clear mempool + self.generate(self.nodes[0], 1) + + # Use default extra_args + self.restart_node(0) + def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + # Test `prioritisetransaction` required parameters assert_raises_rpc_error(-1, "prioritisetransaction", self.nodes[0].prioritisetransaction) assert_raises_rpc_error(-1, "prioritisetransaction", self.nodes[0].prioritisetransaction, '') @@ -44,6 +125,8 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # Test `prioritisetransaction` invalid `fee_delta` assert_raises_rpc_error(-1, "JSON value is not an integer as expected", self.nodes[0].prioritisetransaction, txid=txid, fee_delta='foo') + self.test_diamond() + self.txouts = gen_return_txouts() self.relayfee = self.nodes[0].getnetworkinfo()['relayfee'] diff --git a/test/functional/p2p_blockfilters.py b/test/functional/p2p_blockfilters.py index e73fad439f..e45cef65bd 100755 --- a/test/functional/p2p_blockfilters.py +++ b/test/functional/p2p_blockfilters.py @@ -244,6 +244,12 @@ class CompactFiltersTest(BitcoinTestFramework): peer_0.send_message(request) peer_0.wait_for_disconnect() + self.log.info("Test -peerblockfilters without -blockfilterindex raises an error") + self.stop_node(0) + self.nodes[0].extra_args = ["-peerblockfilters"] + msg = "Error: Cannot set -peerblockfilters without -blockfilterindex." + self.nodes[0].assert_start_raises_init_error(expected_msg=msg) + def compute_last_header(prev_header, hashes): """Compute the last filter header from a starting header and a sequence of filter hashes.""" diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index 6f142f23f2..12ee4b3c27 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -94,7 +94,7 @@ class P2PBlocksOnly(BitcoinTestFramework): self.nodes[0].sendrawtransaction(tx_hex) - # Bump time forward to ensure nNextInvSend timer pops + # Bump time forward to ensure m_next_inv_send_time timer pops self.nodes[0].setmocktime(int(time.time()) + 60) conn.sync_send_with_ping() diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index b264f23fb5..193bd3f1cd 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -200,7 +200,7 @@ class BlockchainTest(BitcoinTestFramework): 'timeout': 0x7fffffffffffffff, # testdummy does not have a timeout so is set to the max int64 value 'min_activation_height': 0, 'status': 'started', - 'status-next': status_next, + 'status_next': status_next, 'since': 144, 'statistics': { 'period': 144, @@ -220,7 +220,7 @@ class BlockchainTest(BitcoinTestFramework): 'timeout': 9223372036854775807, 'min_activation_height': 0, 'status': 'active', - 'status-next': 'active', + 'status_next': 'active', 'since': 0, }, 'height': 0, diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index 1a3d14100f..1695acaaa8 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -18,15 +18,18 @@ from test_framework.util import ( assert_equal, ) from test_framework.wallet_util import bytes_to_wif +from test_framework.wallet import ( + MiniWallet, + getnewdestination, +) class RpcCreateMultiSigTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 self.supports_cli = False - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + if self.is_bdb_compiled(): + self.requires_wallet = True def get_keys(self): self.pub = [] @@ -37,15 +40,20 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): k.generate() self.pub.append(k.get_pubkey().get_bytes().hex()) self.priv.append(bytes_to_wif(k.get_bytes(), k.is_compressed)) - self.final = node2.getnewaddress() + if self.is_bdb_compiled(): + self.final = node2.getnewaddress() + else: + self.final = getnewdestination()[2] def run_test(self): node0, node1, node2 = self.nodes + self.wallet = MiniWallet(test_node=node0) - self.check_addmultisigaddress_errors() + if self.is_bdb_compiled(): + self.check_addmultisigaddress_errors() self.log.info('Generating blocks ...') - self.generate(node0, 149) + self.generate(self.wallet, 149) self.moved = 0 for self.nkeys in [3, 5]: @@ -53,14 +61,14 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): for self.output_type in ["bech32", "p2sh-segwit", "legacy"]: self.get_keys() self.do_multisig() - - self.checkbalances() + if self.is_bdb_compiled(): + self.checkbalances() # Test mixed compressed and uncompressed pubkeys self.log.info('Mixed compressed and uncompressed multisigs are not allowed') - pk0 = node0.getaddressinfo(node0.getnewaddress())['pubkey'] - pk1 = node1.getaddressinfo(node1.getnewaddress())['pubkey'] - pk2 = node2.getaddressinfo(node2.getnewaddress())['pubkey'] + pk0 = getnewdestination()[0].hex() + pk1 = getnewdestination()[0].hex() + pk2 = getnewdestination()[0].hex() # decompress pk2 pk_obj = ECPubKey() @@ -68,26 +76,30 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): pk_obj.compressed = False pk2 = pk_obj.get_bytes().hex() - node0.createwallet(wallet_name='wmulti0', disable_private_keys=True) - wmulti0 = node0.get_wallet_rpc('wmulti0') + if self.is_bdb_compiled(): + node0.createwallet(wallet_name='wmulti0', disable_private_keys=True) + wmulti0 = node0.get_wallet_rpc('wmulti0') # Check all permutations of keys because order matters apparently for keys in itertools.permutations([pk0, pk1, pk2]): # Results should be the same as this legacy one legacy_addr = node0.createmultisig(2, keys, 'legacy')['address'] - result = wmulti0.addmultisigaddress(2, keys, '', 'legacy') - assert_equal(legacy_addr, result['address']) - assert 'warnings' not in result + + if self.is_bdb_compiled(): + result = wmulti0.addmultisigaddress(2, keys, '', 'legacy') + assert_equal(legacy_addr, result['address']) + assert 'warnings' not in result # Generate addresses with the segwit types. These should all make legacy addresses for addr_type in ['bech32', 'p2sh-segwit']: - result = wmulti0.createmultisig(2, keys, addr_type) + result = self.nodes[0].createmultisig(2, keys, addr_type) assert_equal(legacy_addr, result['address']) assert_equal(result['warnings'], ["Unable to make chosen address type, please ensure no uncompressed public keys are present."]) - result = wmulti0.addmultisigaddress(2, keys, '', addr_type) - assert_equal(legacy_addr, result['address']) - assert_equal(result['warnings'], ["Unable to make chosen address type, please ensure no uncompressed public keys are present."]) + if self.is_bdb_compiled(): + result = wmulti0.addmultisigaddress(2, keys, '', addr_type) + assert_equal(legacy_addr, result['address']) + assert_equal(result['warnings'], ["Unable to make chosen address type, please ensure no uncompressed public keys are present."]) self.log.info('Testing sortedmulti descriptors with BIP 67 test vectors') with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/rpc_bip67.json'), encoding='utf-8') as f: @@ -126,26 +138,29 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): bal0 = node0.getbalance() bal1 = node1.getbalance() bal2 = node2.getbalance() + balw = self.wallet.get_balance() height = node0.getblockchaininfo()["blocks"] assert 150 < height < 350 total = 149 * 50 + (height - 149 - 100) * 25 assert bal1 == 0 assert bal2 == self.moved - assert bal0 + bal1 + bal2 == total + assert_equal(bal0 + bal1 + bal2 + balw, total) def do_multisig(self): node0, node1, node2 = self.nodes - if 'wmulti' not in node1.listwallets(): - try: - node1.loadwallet('wmulti') - except JSONRPCException as e: - path = os.path.join(self.options.tmpdir, "node1", "regtest", "wallets", "wmulti") - if e.error['code'] == -18 and "Wallet file verification failed. Failed to load database path '{}'. Path does not exist.".format(path) in e.error['message']: - node1.createwallet(wallet_name='wmulti', disable_private_keys=True) - else: - raise - wmulti = node1.get_wallet_rpc('wmulti') + + if self.is_bdb_compiled(): + if 'wmulti' not in node1.listwallets(): + try: + node1.loadwallet('wmulti') + except JSONRPCException as e: + path = os.path.join(self.options.tmpdir, "node1", "regtest", "wallets", "wmulti") + if e.error['code'] == -18 and "Wallet file verification failed. Failed to load database path '{}'. Path does not exist.".format(path) in e.error['message']: + node1.createwallet(wallet_name='wmulti', disable_private_keys=True) + else: + raise + wmulti = node1.get_wallet_rpc('wmulti') # Construct the expected descriptor desc = 'multi({},{})'.format(self.nsigs, ','.join(self.pub)) @@ -164,17 +179,19 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): if self.output_type == 'bech32': assert madd[0:4] == "bcrt" # actually a bech32 address - # compare against addmultisigaddress - msigw = wmulti.addmultisigaddress(self.nsigs, self.pub, None, self.output_type) - maddw = msigw["address"] - mredeemw = msigw["redeemScript"] - assert_equal(desc, drop_origins(msigw['descriptor'])) - # addmultisigiaddress and createmultisig work the same - assert maddw == madd - assert mredeemw == mredeem - - txid = node0.sendtoaddress(madd, 40) - + if self.is_bdb_compiled(): + # compare against addmultisigaddress + msigw = wmulti.addmultisigaddress(self.nsigs, self.pub, None, self.output_type) + maddw = msigw["address"] + mredeemw = msigw["redeemScript"] + assert_equal(desc, drop_origins(msigw['descriptor'])) + # addmultisigiaddress and createmultisig work the same + assert maddw == madd + assert mredeemw == mredeem + wmulti.unloadwallet() + + spk = bytes.fromhex(node0.validateaddress(madd)["scriptPubKey"]) + txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=spk, amount=1300) tx = node0.getrawtransaction(txid, True) vout = [v["n"] for v in tx["vout"] if madd == v["scriptPubKey"]["address"]] assert len(vout) == 1 @@ -225,8 +242,6 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): txinfo = node0.getrawtransaction(tx, True, blk) self.log.info("n/m=%d/%d %s size=%d vsize=%d weight=%d" % (self.nsigs, self.nkeys, self.output_type, txinfo["size"], txinfo["vsize"], txinfo["weight"])) - wmulti.unloadwallet() - if __name__ == '__main__': RpcCreateMultiSigTest().main() diff --git a/test/functional/rpc_generate.py b/test/functional/rpc_generate.py index 47d7814da3..2b1dd20ea1 100755 --- a/test/functional/rpc_generate.py +++ b/test/functional/rpc_generate.py @@ -2,9 +2,10 @@ # Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test generate RPC.""" +"""Test generate* RPCs.""" from test_framework.test_framework import BitcoinTestFramework +from test_framework.wallet import MiniWallet from test_framework.util import ( assert_equal, assert_raises_rpc_error, @@ -16,6 +17,94 @@ class RPCGenerateTest(BitcoinTestFramework): self.num_nodes = 1 def run_test(self): + self.test_generatetoaddress() + self.test_generate() + self.test_generateblock() + + def test_generatetoaddress(self): + self.generatetoaddress(self.nodes[0], 1, 'mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ') + assert_raises_rpc_error(-5, "Invalid address", self.generatetoaddress, self.nodes[0], 1, '3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy') + + def test_generateblock(self): + node = self.nodes[0] + miniwallet = MiniWallet(node) + miniwallet.rescan_utxos() + + self.log.info('Generate an empty block to address') + address = miniwallet.get_address() + hash = self.generateblock(node, output=address, transactions=[])['hash'] + block = node.getblock(blockhash=hash, verbose=2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], address) + + self.log.info('Generate an empty block to a descriptor') + hash = self.generateblock(node, 'addr(' + address + ')', [])['hash'] + block = node.getblock(blockhash=hash, verbosity=2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], address) + + self.log.info('Generate an empty block to a combo descriptor with compressed pubkey') + combo_key = '0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798' + combo_address = 'bcrt1qw508d6qejxtdg4y5r3zarvary0c5xw7kygt080' + hash = self.generateblock(node, 'combo(' + combo_key + ')', [])['hash'] + block = node.getblock(hash, 2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], combo_address) + + self.log.info('Generate an empty block to a combo descriptor with uncompressed pubkey') + combo_key = '0408ef68c46d20596cc3f6ddf7c8794f71913add807f1dc55949fa805d764d191c0b7ce6894c126fce0babc6663042f3dde9b0cf76467ea315514e5a6731149c67' + combo_address = 'mkc9STceoCcjoXEXe6cm66iJbmjM6zR9B2' + hash = self.generateblock(node, 'combo(' + combo_key + ')', [])['hash'] + block = node.getblock(hash, 2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], combo_address) + + # Generate some extra mempool transactions to verify they don't get mined + for _ in range(10): + miniwallet.send_self_transfer(from_node=node) + + self.log.info('Generate block with txid') + txid = miniwallet.send_self_transfer(from_node=node)['txid'] + hash = self.generateblock(node, address, [txid])['hash'] + block = node.getblock(hash, 1) + assert_equal(len(block['tx']), 2) + assert_equal(block['tx'][1], txid) + + self.log.info('Generate block with raw tx') + rawtx = miniwallet.create_self_transfer()['hex'] + hash = self.generateblock(node, address, [rawtx])['hash'] + + block = node.getblock(hash, 1) + assert_equal(len(block['tx']), 2) + txid = block['tx'][1] + assert_equal(node.getrawtransaction(txid=txid, verbose=False, blockhash=hash), rawtx) + + self.log.info('Fail to generate block with out of order txs') + txid1 = miniwallet.send_self_transfer(from_node=node)['txid'] + utxo1 = miniwallet.get_utxo(txid=txid1) + rawtx2 = miniwallet.create_self_transfer(utxo_to_spend=utxo1)['hex'] + assert_raises_rpc_error(-25, 'TestBlockValidity failed: bad-txns-inputs-missingorspent', self.generateblock, node, address, [rawtx2, txid1]) + + self.log.info('Fail to generate block with txid not in mempool') + missing_txid = '0000000000000000000000000000000000000000000000000000000000000000' + assert_raises_rpc_error(-5, 'Transaction ' + missing_txid + ' not in mempool.', self.generateblock, node, address, [missing_txid]) + + self.log.info('Fail to generate block with invalid raw tx') + invalid_raw_tx = '0000' + assert_raises_rpc_error(-22, 'Transaction decode failed for ' + invalid_raw_tx, self.generateblock, node, address, [invalid_raw_tx]) + + self.log.info('Fail to generate block with invalid address/descriptor') + assert_raises_rpc_error(-5, 'Invalid address or descriptor', self.generateblock, node, '1234', []) + + self.log.info('Fail to generate block with a ranged descriptor') + ranged_descriptor = 'pkh(tpubD6NzVbkrYhZ4XgiXtGrdW5XDAPFCL9h7we1vwNCpn8tGbBcgfVYjXyhWo4E1xkh56hjod1RhGjxbaTLV3X4FyWuejifB9jusQ46QzG87VKp/0/*)' + assert_raises_rpc_error(-8, 'Ranged descriptor not accepted. Maybe pass through deriveaddresses first?', self.generateblock, node, ranged_descriptor, []) + + self.log.info('Fail to generate block with a descriptor missing a private key') + child_descriptor = 'pkh(tpubD6NzVbkrYhZ4XgiXtGrdW5XDAPFCL9h7we1vwNCpn8tGbBcgfVYjXyhWo4E1xkh56hjod1RhGjxbaTLV3X4FyWuejifB9jusQ46QzG87VKp/0\'/0)' + assert_raises_rpc_error(-5, 'Cannot derive script without private keys', self.generateblock, node, child_descriptor, []) + + def test_generate(self): message = ( "generate\n\n" "has been replaced by the -generate " diff --git a/test/functional/rpc_generateblock.py b/test/functional/rpc_generateblock.py deleted file mode 100755 index 7eeb745817..0000000000 --- a/test/functional/rpc_generateblock.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2020-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -'''Test generateblock rpc. -''' - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.wallet import MiniWallet -from test_framework.util import ( - assert_equal, - assert_raises_rpc_error, -) - - -class GenerateBlockTest(BitcoinTestFramework): - def set_test_params(self): - self.num_nodes = 1 - - def run_test(self): - node = self.nodes[0] - miniwallet = MiniWallet(node) - miniwallet.rescan_utxos() - - self.log.info('Generate an empty block to address') - address = miniwallet.get_address() - hash = self.generateblock(node, output=address, transactions=[])['hash'] - block = node.getblock(blockhash=hash, verbose=2) - assert_equal(len(block['tx']), 1) - assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], address) - - self.log.info('Generate an empty block to a descriptor') - hash = self.generateblock(node, 'addr(' + address + ')', [])['hash'] - block = node.getblock(blockhash=hash, verbosity=2) - assert_equal(len(block['tx']), 1) - assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], address) - - self.log.info('Generate an empty block to a combo descriptor with compressed pubkey') - combo_key = '0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798' - combo_address = 'bcrt1qw508d6qejxtdg4y5r3zarvary0c5xw7kygt080' - hash = self.generateblock(node, 'combo(' + combo_key + ')', [])['hash'] - block = node.getblock(hash, 2) - assert_equal(len(block['tx']), 1) - assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], combo_address) - - self.log.info('Generate an empty block to a combo descriptor with uncompressed pubkey') - combo_key = '0408ef68c46d20596cc3f6ddf7c8794f71913add807f1dc55949fa805d764d191c0b7ce6894c126fce0babc6663042f3dde9b0cf76467ea315514e5a6731149c67' - combo_address = 'mkc9STceoCcjoXEXe6cm66iJbmjM6zR9B2' - hash = self.generateblock(node, 'combo(' + combo_key + ')', [])['hash'] - block = node.getblock(hash, 2) - assert_equal(len(block['tx']), 1) - assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], combo_address) - - # Generate some extra mempool transactions to verify they don't get mined - for _ in range(10): - miniwallet.send_self_transfer(from_node=node) - - self.log.info('Generate block with txid') - txid = miniwallet.send_self_transfer(from_node=node)['txid'] - hash = self.generateblock(node, address, [txid])['hash'] - block = node.getblock(hash, 1) - assert_equal(len(block['tx']), 2) - assert_equal(block['tx'][1], txid) - - self.log.info('Generate block with raw tx') - rawtx = miniwallet.create_self_transfer()['hex'] - hash = self.generateblock(node, address, [rawtx])['hash'] - - block = node.getblock(hash, 1) - assert_equal(len(block['tx']), 2) - txid = block['tx'][1] - assert_equal(node.getrawtransaction(txid=txid, verbose=False, blockhash=hash), rawtx) - - self.log.info('Fail to generate block with out of order txs') - txid1 = miniwallet.send_self_transfer(from_node=node)['txid'] - utxo1 = miniwallet.get_utxo(txid=txid1) - rawtx2 = miniwallet.create_self_transfer(utxo_to_spend=utxo1)['hex'] - assert_raises_rpc_error(-25, 'TestBlockValidity failed: bad-txns-inputs-missingorspent', self.generateblock, node, address, [rawtx2, txid1]) - - self.log.info('Fail to generate block with txid not in mempool') - missing_txid = '0000000000000000000000000000000000000000000000000000000000000000' - assert_raises_rpc_error(-5, 'Transaction ' + missing_txid + ' not in mempool.', self.generateblock, node, address, [missing_txid]) - - self.log.info('Fail to generate block with invalid raw tx') - invalid_raw_tx = '0000' - assert_raises_rpc_error(-22, 'Transaction decode failed for ' + invalid_raw_tx, self.generateblock, node, address, [invalid_raw_tx]) - - self.log.info('Fail to generate block with invalid address/descriptor') - assert_raises_rpc_error(-5, 'Invalid address or descriptor', self.generateblock, node, '1234', []) - - self.log.info('Fail to generate block with a ranged descriptor') - ranged_descriptor = 'pkh(tpubD6NzVbkrYhZ4XgiXtGrdW5XDAPFCL9h7we1vwNCpn8tGbBcgfVYjXyhWo4E1xkh56hjod1RhGjxbaTLV3X4FyWuejifB9jusQ46QzG87VKp/0/*)' - assert_raises_rpc_error(-8, 'Ranged descriptor not accepted. Maybe pass through deriveaddresses first?', self.generateblock, node, ranged_descriptor, []) - - self.log.info('Fail to generate block with a descriptor missing a private key') - child_descriptor = 'pkh(tpubD6NzVbkrYhZ4XgiXtGrdW5XDAPFCL9h7we1vwNCpn8tGbBcgfVYjXyhWo4E1xkh56hjod1RhGjxbaTLV3X4FyWuejifB9jusQ46QzG87VKp/0\'/0)' - assert_raises_rpc_error(-5, 'Cannot derive script without private keys', self.generateblock, node, child_descriptor, []) - -if __name__ == '__main__': - GenerateBlockTest().main() diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 210025104e..8651bcf636 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -573,17 +573,17 @@ def create_lots_of_big_transactions(node, txouts, utxos, num, fee): return txids -def mine_large_block(test_framework, node, utxos=None): +def mine_large_block(test_framework, mini_wallet, node): # generate a 66k transaction, # and 14 of them is close to the 1MB block limit - num = 14 txouts = gen_return_txouts() - utxos = utxos if utxos is not None else [] - if len(utxos) < num: - utxos.clear() - utxos.extend(node.listunspent()) - fee = 100 * node.getnetworkinfo()["relayfee"] - create_lots_of_big_transactions(node, txouts, utxos, num, fee=fee) + from .messages import COIN + fee = 100 * int(node.getnetworkinfo()["relayfee"] * COIN) + for _ in range(14): + tx = mini_wallet.create_self_transfer(from_node=node, fee_rate=0, mempool_valid=False)['tx'] + tx.vout[0].nValue -= fee + tx.vout.extend(txouts) + mini_wallet.sendrawtransaction(from_node=node, tx_hex=tx.serialize().hex()) test_framework.generate(node, 1) diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index dd41a740ae..e86f365f11 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -8,7 +8,10 @@ from copy import deepcopy from decimal import Decimal from enum import Enum from random import choice -from typing import Optional +from typing import ( + Any, + Optional, +) from test_framework.address import ( base58_to_byte, create_deterministic_address_bcrt1_p2tr_op_true, @@ -93,6 +96,9 @@ class MiniWallet: self._address, self._internal_key = create_deterministic_address_bcrt1_p2tr_op_true() self._scriptPubKey = bytes.fromhex(self._test_node.validateaddress(self._address)['scriptPubKey']) + def get_balance(self): + return sum(u['value'] for u in self._utxos) + def rescan_utxos(self): """Drop all utxos and rescan the utxo set""" self._utxos = [] @@ -131,24 +137,30 @@ class MiniWallet: self._utxos.append({'txid': cb_tx['txid'], 'vout': 0, 'value': cb_tx['vout'][0]['value'], 'height': block_info['height']}) return blocks + def get_scriptPubKey(self): + return self._scriptPubKey + def get_descriptor(self): return descsum_create(f'raw({self._scriptPubKey.hex()})') def get_address(self): return self._address - def get_utxo(self, *, txid: Optional[str]='', mark_as_spent=True): + def get_utxo(self, *, txid: str = '', vout: Optional[int] = None, mark_as_spent=True): """ Returns a utxo and marks it as spent (pops it from the internal list) Args: txid: get the first utxo we find from a specific transaction """ - index = -1 # by default the last utxo self._utxos = sorted(self._utxos, key=lambda k: (k['value'], -k['height'])) # Put the largest utxo last if txid: - utxo = next(filter(lambda utxo: txid == utxo['txid'], self._utxos)) - index = self._utxos.index(utxo) + utxo_filter: Any = filter(lambda utxo: txid == utxo['txid'], self._utxos) + else: + utxo_filter = reversed(self._utxos) # By default the largest utxo + if vout is not None: + utxo_filter = filter(lambda utxo: vout == utxo['vout'], utxo_filter) + index = self._utxos.index(next(utxo_filter)) if mark_as_spent: return self._utxos.pop(index) else: @@ -179,6 +191,47 @@ class MiniWallet: txid = self.sendrawtransaction(from_node=from_node, tx_hex=tx.serialize().hex()) return txid, 1 + def send_self_transfer_multi(self, **kwargs): + """ + Create and send a transaction that spends the given UTXOs and creates a + certain number of outputs with equal amounts. + + Returns a dictionary with + - txid + - serialized transaction in hex format + - transaction as CTransaction instance + - list of newly created UTXOs, ordered by vout index + """ + tx = self.create_self_transfer_multi(**kwargs) + txid = self.sendrawtransaction(from_node=kwargs['from_node'], tx_hex=tx.serialize().hex()) + return {'new_utxos': [self.get_utxo(txid=txid, vout=vout) for vout in range(len(tx.vout))], + 'txid': txid, 'hex': tx.serialize().hex(), 'tx': tx} + + def create_self_transfer_multi(self, *, from_node, utxos_to_spend=None, num_outputs=1, fee_per_output=1000): + """ + Create and return a transaction that spends the given UTXOs and creates a + certain number of outputs with equal amounts. + """ + utxos_to_spend = utxos_to_spend or [self.get_utxo()] + # create simple tx template (1 input, 1 output) + tx = self.create_self_transfer(fee_rate=0, from_node=from_node, utxo_to_spend=utxos_to_spend[0], mempool_valid=False)['tx'] + + # duplicate inputs, witnesses and outputs + tx.vin = [deepcopy(tx.vin[0]) for _ in range(len(utxos_to_spend))] + tx.wit.vtxinwit = [deepcopy(tx.wit.vtxinwit[0]) for _ in range(len(utxos_to_spend))] + tx.vout = [deepcopy(tx.vout[0]) for _ in range(num_outputs)] + + # adapt input prevouts + for i, utxo in enumerate(utxos_to_spend): + tx.vin[i] = CTxIn(COutPoint(int(utxo['txid'], 16), utxo['vout'])) + + # adapt output amounts (use fixed fee per output) + inputs_value_total = sum([int(COIN * utxo['value']) for utxo in utxos_to_spend]) + outputs_value_total = inputs_value_total - fee_per_output * num_outputs + for o in tx.vout: + o.nValue = outputs_value_total // num_outputs + return tx + def create_self_transfer(self, *, fee_rate=Decimal("0.003"), from_node=None, utxo_to_spend=None, mempool_valid=True, locktime=0, sequence=0): """Create and return a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed.""" from_node = from_node or self._test_node diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 93bd61bac1..381d2b9426 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -188,8 +188,7 @@ BASE_SCRIPTS = [ 'rpc_decodescript.py', 'rpc_blockchain.py', 'rpc_deprecated.py', - 'wallet_disable.py --legacy-wallet', - 'wallet_disable.py --descriptors', + 'wallet_disable.py', 'p2p_addr_relay.py', 'p2p_getaddr_caching.py', 'p2p_getdata.py', @@ -225,8 +224,7 @@ BASE_SCRIPTS = [ 'feature_rbf.py --descriptors', 'mempool_packages.py', 'mempool_package_onemore.py', - 'rpc_createmultisig.py --legacy-wallet', - 'rpc_createmultisig.py --descriptors', + 'rpc_createmultisig.py', 'rpc_packages.py', 'mempool_package_limits.py', 'feature_versionbits_warning.py', @@ -237,7 +235,6 @@ BASE_SCRIPTS = [ 'p2p_eviction.py', 'wallet_signmessagewithaddress.py', 'rpc_signmessagewithprivkey.py', - 'rpc_generateblock.py', 'rpc_generate.py', 'wallet_balance.py --legacy-wallet', 'wallet_balance.py --descriptors', @@ -280,6 +277,8 @@ BASE_SCRIPTS = [ 'wallet_create_tx.py --legacy-wallet', 'wallet_send.py --legacy-wallet', 'wallet_send.py --descriptors', + 'wallet_sendall.py --legacy-wallet', + 'wallet_sendall.py --descriptors', 'wallet_create_tx.py --descriptors', 'wallet_taproot.py', 'wallet_inactive_hdchains.py', @@ -310,8 +309,7 @@ BASE_SCRIPTS = [ 'feature_unsupported_utxo_db.py', 'feature_logging.py', 'feature_anchors.py', - 'feature_coinstatsindex.py --legacy-wallet', - 'feature_coinstatsindex.py --descriptors', + 'feature_coinstatsindex.py', 'wallet_orphanedreward.py', 'wallet_timelock.py', 'p2p_node_network_limited.py', @@ -590,11 +588,12 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= # Clean up dangling processes if any. This may only happen with --failfast option. # Killing the process group will also terminate the current process but that is # not an issue - if len(job_queue.jobs): + if not os.getenv("CI_FAILFAST_TEST_LEAVE_DANGLING") and len(job_queue.jobs): os.killpg(os.getpgid(0), signal.SIGKILL) sys.exit(not all_passed) + def print_results(test_results, max_len_name, runtime): results = "\n" + BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "STATUS ", "DURATION") + BOLD[0] diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index 0cfbefb719..0c93821e7f 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -50,7 +50,9 @@ class WalletTest(BitcoinTestFramework): self.num_nodes = 2 self.setup_clean_chain = True self.extra_args = [ - ['-limitdescendantcount=3'], # Limit mempool descendants as a hack to have wallet txs rejected from the mempool + # Limit mempool descendants as a hack to have wallet txs rejected from the mempool. + # Set walletrejectlongchains=0 so the wallet still creates the transactions. + ['-limitdescendantcount=3', '-walletrejectlongchains=0'], [], ] diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index a7873838be..a6c93ba5f9 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -25,7 +25,7 @@ class WalletTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 self.extra_args = [[ - "-acceptnonstdtxn=1", + "-acceptnonstdtxn=1", "-walletrejectlongchains=0" ]] * self.num_nodes self.setup_clean_chain = True self.supports_cli = False @@ -142,7 +142,7 @@ class WalletTest(BitcoinTestFramework): self.nodes[2].lockunspent(False, [unspent_0], True) # Restarting the node with the lock written to the wallet should keep the lock - self.restart_node(2) + self.restart_node(2, ["-walletrejectlongchains=0"]) assert_raises_rpc_error(-8, "Invalid parameter, output already locked", self.nodes[2].lockunspent, False, [unspent_0]) # Unloading and reloading the wallet with a persistent lock should keep the lock @@ -568,7 +568,7 @@ class WalletTest(BitcoinTestFramework): self.log.info("Test -reindex") self.stop_nodes() # set lower ancestor limit for later - self.start_node(0, ['-reindex', "-limitancestorcount=" + str(chainlimit)]) + self.start_node(0, ['-reindex', "-walletrejectlongchains=0", "-limitancestorcount=" + str(chainlimit)]) self.start_node(1, ['-reindex', "-limitancestorcount=" + str(chainlimit)]) self.start_node(2, ['-reindex', "-limitancestorcount=" + str(chainlimit)]) # reindex will leave rpc warm up "early"; Wait for it to finish diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index f6843d597d..3b23ee8e94 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -442,7 +442,9 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): self.generate(peer_node, 1) # Create single-input PSBT for transaction to be bumped - psbt = watcher.walletcreatefundedpsbt([], {dest_address: 0.0005}, 0, {"fee_rate": 1}, True)['psbt'] + # Ensure the payment amount + change can be fully funded using one of the 0.001BTC inputs. + psbt = watcher.walletcreatefundedpsbt([watcher.listunspent()[0]], {dest_address: 0.0005}, 0, + {"fee_rate": 1, "add_inputs": False}, True)['psbt'] psbt_signed = signer.walletprocesspsbt(psbt=psbt, sign=True, sighashtype="ALL", bip32derivs=True) psbt_final = watcher.finalizepsbt(psbt_signed["psbt"]) original_txid = watcher.sendrawtransaction(psbt_final["hex"]) diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py index 4416a9655f..e8234de032 100755 --- a/test/functional/wallet_createwallet.py +++ b/test/functional/wallet_createwallet.py @@ -164,5 +164,10 @@ class CreateWalletTest(BitcoinTestFramework): self.log.info('Using a passphrase with private keys disabled returns error') assert_raises_rpc_error(-4, 'Passphrase provided but private keys are disabled. A passphrase is only used to encrypt private keys, so cannot be used for wallets with private keys disabled.', self.nodes[0].createwallet, wallet_name='w9', disable_private_keys=True, passphrase='thisisapassphrase') + if self.is_bdb_compiled(): + self.log.info("Test legacy wallet deprecation") + res = self.nodes[0].createwallet(wallet_name="legacy_w0", descriptors=False, passphrase=None) + assert_equal(res["warning"], "Wallet created successfully. The legacy wallet type is being deprecated and support for creating and opening legacy wallets will be removed in the future.") + if __name__ == '__main__': CreateWalletTest().main() diff --git a/test/functional/wallet_disable.py b/test/functional/wallet_disable.py index 2c7996ca6b..74cddf2738 100755 --- a/test/functional/wallet_disable.py +++ b/test/functional/wallet_disable.py @@ -26,10 +26,6 @@ class DisableWalletTest (BitcoinTestFramework): x = self.nodes[0].validateaddress('mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ') assert x['isvalid'] == True - # Checking mining to an address without a wallet. Generating to a valid address should succeed - # but generating to an invalid address will fail. - self.generatetoaddress(self.nodes[0], 1, 'mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ') - assert_raises_rpc_error(-5, "Invalid address", self.generatetoaddress, self.nodes[0], 1, '3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy') if __name__ == '__main__': - DisableWalletTest ().main () + DisableWalletTest().main() diff --git a/test/functional/wallet_importprunedfunds.py b/test/functional/wallet_importprunedfunds.py index cdb5823109..2a4d0981c7 100755 --- a/test/functional/wallet_importprunedfunds.py +++ b/test/functional/wallet_importprunedfunds.py @@ -5,9 +5,13 @@ """Test the importprunedfunds and removeprunedfunds RPCs.""" from decimal import Decimal -from test_framework.blocktools import COINBASE_MATURITY from test_framework.address import key_to_p2wpkh +from test_framework.blocktools import COINBASE_MATURITY from test_framework.key import ECKey +from test_framework.messages import ( + CMerkleBlock, + from_hex, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -15,6 +19,7 @@ from test_framework.util import ( ) from test_framework.wallet_util import bytes_to_wif + class ImportPrunedFundsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -124,5 +129,18 @@ class ImportPrunedFundsTest(BitcoinTestFramework): w1.removeprunedfunds(txnid3) assert not [tx for tx in w1.listtransactions(include_watchonly=True) if tx['txid'] == txnid3] + # Check various RPC parameter validation errors + assert_raises_rpc_error(-22, "TX decode failed", w1.importprunedfunds, b'invalid tx'.hex(), proof1) + assert_raises_rpc_error(-5, "Transaction given doesn't exist in proof", w1.importprunedfunds, rawtxn2, proof1) + + mb = from_hex(CMerkleBlock(), proof1) + mb.header.hashMerkleRoot = 0xdeadbeef # cause mismatch between merkle root and merkle block + assert_raises_rpc_error(-5, "Something wrong with merkleblock", w1.importprunedfunds, rawtxn1, mb.serialize().hex()) + + mb = from_hex(CMerkleBlock(), proof1) + mb.header.nTime += 1 # modify arbitrary block header field to change block hash + assert_raises_rpc_error(-5, "Block not found in chain", w1.importprunedfunds, rawtxn1, mb.serialize().hex()) + + if __name__ == '__main__': ImportPrunedFundsTest().main() diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index 5aae2c813a..6552bfe60c 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -38,7 +38,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): # Can take a few seconds due to transaction trickling peer_first.wait_for_broadcast([txid]) - # Add a second peer since txs aren't rebroadcast to the same peer (see filterInventoryKnown) + # Add a second peer since txs aren't rebroadcast to the same peer (see m_tx_inventory_known_filter) peer_second = node.add_p2p_connection(P2PTxInvStore()) self.log.info("Create a block") diff --git a/test/functional/wallet_sendall.py b/test/functional/wallet_sendall.py new file mode 100755 index 0000000000..aa8d2a9d2c --- /dev/null +++ b/test/functional/wallet_sendall.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the sendall RPC command.""" + +from decimal import Decimal, getcontext + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_rpc_error, +) + +# Decorator to reset activewallet to zero utxos +def cleanup(func): + def wrapper(self): + try: + func(self) + finally: + if 0 < self.wallet.getbalances()["mine"]["trusted"]: + self.wallet.sendall([self.remainder_target]) + assert_equal(0, self.wallet.getbalances()["mine"]["trusted"]) # wallet is empty + return wrapper + +class SendallTest(BitcoinTestFramework): + # Setup and helpers + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def set_test_params(self): + getcontext().prec=10 + self.num_nodes = 1 + self.setup_clean_chain = True + + def assert_balance_swept_completely(self, tx, balance): + output_sum = sum([o["value"] for o in tx["decoded"]["vout"]]) + assert_equal(output_sum, balance + tx["fee"]) + assert_equal(0, self.wallet.getbalances()["mine"]["trusted"]) # wallet is empty + + def assert_tx_has_output(self, tx, addr, value=None): + for output in tx["decoded"]["vout"]: + if addr == output["scriptPubKey"]["address"] and value is None or value == output["value"]: + return + raise AssertionError("Output to {} not present or wrong amount".format(addr)) + + def assert_tx_has_outputs(self, tx, expected_outputs): + assert_equal(len(expected_outputs), len(tx["decoded"]["vout"])) + for eo in expected_outputs: + self.assert_tx_has_output(tx, eo["address"], eo["value"]) + + def add_utxos(self, amounts): + for a in amounts: + self.def_wallet.sendtoaddress(self.wallet.getnewaddress(), a) + self.generate(self.nodes[0], 1) + assert_greater_than(self.wallet.getbalances()["mine"]["trusted"], 0) + return self.wallet.getbalances()["mine"]["trusted"] + + # Helper schema for success cases + def test_sendall_success(self, sendall_args, remaining_balance = 0): + sendall_tx_receipt = self.wallet.sendall(sendall_args) + self.generate(self.nodes[0], 1) + # wallet has remaining balance (usually empty) + assert_equal(remaining_balance, self.wallet.getbalances()["mine"]["trusted"]) + + assert_equal(sendall_tx_receipt["complete"], True) + return self.wallet.gettransaction(txid = sendall_tx_receipt["txid"], verbose = True) + + @cleanup + def gen_and_clean(self): + self.add_utxos([15, 2, 4]) + + def test_cleanup(self): + self.log.info("Test that cleanup wrapper empties wallet") + self.gen_and_clean() + assert_equal(0, self.wallet.getbalances()["mine"]["trusted"]) # wallet is empty + + # Actual tests + @cleanup + def sendall_two_utxos(self): + self.log.info("Testing basic sendall case without specific amounts") + pre_sendall_balance = self.add_utxos([10,11]) + tx_from_wallet = self.test_sendall_success(sendall_args = [self.remainder_target]) + + self.assert_tx_has_outputs(tx = tx_from_wallet, + expected_outputs = [ + { "address": self.remainder_target, "value": pre_sendall_balance + tx_from_wallet["fee"] } # fee is neg + ] + ) + self.assert_balance_swept_completely(tx_from_wallet, pre_sendall_balance) + + @cleanup + def sendall_split(self): + self.log.info("Testing sendall where two recipients have unspecified amount") + pre_sendall_balance = self.add_utxos([1, 2, 3, 15]) + tx_from_wallet = self.test_sendall_success([self.remainder_target, self.split_target]) + + half = (pre_sendall_balance + tx_from_wallet["fee"]) / 2 + self.assert_tx_has_outputs(tx_from_wallet, + expected_outputs = [ + { "address": self.split_target, "value": half }, + { "address": self.remainder_target, "value": half } + ] + ) + self.assert_balance_swept_completely(tx_from_wallet, pre_sendall_balance) + + @cleanup + def sendall_and_spend(self): + self.log.info("Testing sendall in combination with paying specified amount to recipient") + pre_sendall_balance = self.add_utxos([8, 13]) + tx_from_wallet = self.test_sendall_success([{self.recipient: 5}, self.remainder_target]) + + self.assert_tx_has_outputs(tx_from_wallet, + expected_outputs = [ + { "address": self.recipient, "value": 5 }, + { "address": self.remainder_target, "value": pre_sendall_balance - 5 + tx_from_wallet["fee"] } + ] + ) + self.assert_balance_swept_completely(tx_from_wallet, pre_sendall_balance) + + @cleanup + def sendall_invalid_recipient_addresses(self): + self.log.info("Test having only recipient with specified amount, missing recipient with unspecified amount") + self.add_utxos([12, 9]) + + assert_raises_rpc_error( + -8, + "Must provide at least one address without a specified amount" , + self.wallet.sendall, + [{self.recipient: 5}] + ) + + @cleanup + def sendall_duplicate_recipient(self): + self.log.info("Test duplicate destination") + self.add_utxos([1, 8, 3, 9]) + + assert_raises_rpc_error( + -8, + "Invalid parameter, duplicated address: {}".format(self.remainder_target), + self.wallet.sendall, + [self.remainder_target, self.remainder_target] + ) + + @cleanup + def sendall_invalid_amounts(self): + self.log.info("Test sending more than balance") + pre_sendall_balance = self.add_utxos([7, 14]) + + expected_tx = self.wallet.sendall(recipients=[{self.recipient: 5}, self.remainder_target], options={"add_to_wallet": False}) + tx = self.wallet.decoderawtransaction(expected_tx['hex']) + fee = 21 - sum([o["value"] for o in tx["vout"]]) + + assert_raises_rpc_error(-6, "Assigned more value to outputs than available funds.", self.wallet.sendall, + [{self.recipient: pre_sendall_balance + 1}, self.remainder_target]) + assert_raises_rpc_error(-6, "Insufficient funds for fees after creating specified outputs.", self.wallet.sendall, + [{self.recipient: pre_sendall_balance}, self.remainder_target]) + assert_raises_rpc_error(-8, "Specified output amount to {} is below dust threshold".format(self.recipient), + self.wallet.sendall, [{self.recipient: 0.00000001}, self.remainder_target]) + assert_raises_rpc_error(-6, "Dynamically assigned remainder results in dust output.", self.wallet.sendall, + [{self.recipient: pre_sendall_balance - fee}, self.remainder_target]) + assert_raises_rpc_error(-6, "Dynamically assigned remainder results in dust output.", self.wallet.sendall, + [{self.recipient: pre_sendall_balance - fee - Decimal(0.00000010)}, self.remainder_target]) + + # @cleanup not needed because different wallet used + def sendall_negative_effective_value(self): + self.log.info("Test that sendall fails if all UTXOs have negative effective value") + # Use dedicated wallet for dust amounts and unload wallet at end + self.nodes[0].createwallet("dustwallet") + dust_wallet = self.nodes[0].get_wallet_rpc("dustwallet") + + self.def_wallet.sendtoaddress(dust_wallet.getnewaddress(), 0.00000400) + self.def_wallet.sendtoaddress(dust_wallet.getnewaddress(), 0.00000300) + self.generate(self.nodes[0], 1) + assert_greater_than(dust_wallet.getbalances()["mine"]["trusted"], 0) + + assert_raises_rpc_error(-6, "Total value of UTXO pool too low to pay for transaction." + + " Try using lower feerate or excluding uneconomic UTXOs with 'send_max' option.", + dust_wallet.sendall, recipients=[self.remainder_target], fee_rate=300) + + dust_wallet.unloadwallet() + + @cleanup + def sendall_with_send_max(self): + self.log.info("Check that `send_max` option causes negative value UTXOs to be left behind") + self.add_utxos([0.00000400, 0.00000300, 1]) + + # sendall with send_max + sendall_tx_receipt = self.wallet.sendall(recipients=[self.remainder_target], fee_rate=300, options={"send_max": True}) + tx_from_wallet = self.wallet.gettransaction(txid = sendall_tx_receipt["txid"], verbose = True) + + assert_equal(len(tx_from_wallet["decoded"]["vin"]), 1) + self.assert_tx_has_outputs(tx_from_wallet, [{"address": self.remainder_target, "value": 1 + tx_from_wallet["fee"]}]) + assert_equal(self.wallet.getbalances()["mine"]["trusted"], Decimal("0.00000700")) + + self.def_wallet.sendtoaddress(self.wallet.getnewaddress(), 1) + self.generate(self.nodes[0], 1) + + @cleanup + def sendall_specific_inputs(self): + self.log.info("Test sendall with a subset of UTXO pool") + self.add_utxos([17, 4]) + utxo = self.wallet.listunspent()[0] + + sendall_tx_receipt = self.wallet.sendall(recipients=[self.remainder_target], options={"inputs": [utxo]}) + tx_from_wallet = self.wallet.gettransaction(txid = sendall_tx_receipt["txid"], verbose = True) + assert_equal(len(tx_from_wallet["decoded"]["vin"]), 1) + assert_equal(len(tx_from_wallet["decoded"]["vout"]), 1) + assert_equal(tx_from_wallet["decoded"]["vin"][0]["txid"], utxo["txid"]) + assert_equal(tx_from_wallet["decoded"]["vin"][0]["vout"], utxo["vout"]) + self.assert_tx_has_output(tx_from_wallet, self.remainder_target) + + self.generate(self.nodes[0], 1) + assert_greater_than(self.wallet.getbalances()["mine"]["trusted"], 0) + + @cleanup + def sendall_fails_on_missing_input(self): + # fails because UTXO was previously spent, and wallet is empty + self.log.info("Test sendall fails because specified UTXO is not available") + self.add_utxos([16, 5]) + spent_utxo = self.wallet.listunspent()[0] + + # fails on unconfirmed spent UTXO + self.wallet.sendall(recipients=[self.remainder_target]) + assert_raises_rpc_error(-8, + "Input not available. UTXO ({}:{}) was already spent.".format(spent_utxo["txid"], spent_utxo["vout"]), + self.wallet.sendall, recipients=[self.remainder_target], options={"inputs": [spent_utxo]}) + + # fails on specific previously spent UTXO, while other UTXOs exist + self.generate(self.nodes[0], 1) + self.add_utxos([19, 2]) + assert_raises_rpc_error(-8, + "Input not available. UTXO ({}:{}) was already spent.".format(spent_utxo["txid"], spent_utxo["vout"]), + self.wallet.sendall, recipients=[self.remainder_target], options={"inputs": [spent_utxo]}) + + # fails because UTXO is unknown, while other UTXOs exist + foreign_utxo = self.def_wallet.listunspent()[0] + assert_raises_rpc_error(-8, "Input not found. UTXO ({}:{}) is not part of wallet.".format(foreign_utxo["txid"], + foreign_utxo["vout"]), self.wallet.sendall, recipients=[self.remainder_target], + options={"inputs": [foreign_utxo]}) + + @cleanup + def sendall_fails_on_no_address(self): + self.log.info("Test sendall fails because no address is provided") + self.add_utxos([19, 2]) + + assert_raises_rpc_error( + -8, + "Must provide at least one address without a specified amount" , + self.wallet.sendall, + [] + ) + + @cleanup + def sendall_fails_on_specific_inputs_with_send_max(self): + self.log.info("Test sendall fails because send_max is used while specific inputs are provided") + self.add_utxos([15, 6]) + utxo = self.wallet.listunspent()[0] + + assert_raises_rpc_error(-8, + "Cannot combine send_max with specific inputs.", + self.wallet.sendall, + recipients=[self.remainder_target], + options={"inputs": [utxo], "send_max": True}) + + def run_test(self): + self.nodes[0].createwallet("activewallet") + self.wallet = self.nodes[0].get_wallet_rpc("activewallet") + self.def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + self.generate(self.nodes[0], 101) + self.recipient = self.def_wallet.getnewaddress() # payee for a specific amount + self.remainder_target = self.def_wallet.getnewaddress() # address that receives everything left after payments and fees + self.split_target = self.def_wallet.getnewaddress() # 2nd target when splitting rest + + # Test cleanup + self.test_cleanup() + + # Basic sweep: everything to one address + self.sendall_two_utxos() + + # Split remainder to two addresses with equal amounts + self.sendall_split() + + # Pay recipient and sweep remainder + self.sendall_and_spend() + + # sendall fails if no recipient has unspecified amount + self.sendall_invalid_recipient_addresses() + + # Sendall fails if same destination is provided twice + self.sendall_duplicate_recipient() + + # Sendall fails when trying to spend more than the balance + self.sendall_invalid_amounts() + + # Sendall fails when wallet has no economically spendable UTXOs + self.sendall_negative_effective_value() + + # Leave dust behind if using send_max + self.sendall_with_send_max() + + # Sendall succeeds with specific inputs + self.sendall_specific_inputs() + + # Fails for the right reasons on missing or previously spent UTXOs + self.sendall_fails_on_missing_input() + + # Sendall fails when no address is provided + self.sendall_fails_on_no_address() + + # Sendall fails when using send_max while specifying inputs + self.sendall_fails_on_specific_inputs_with_send_max() + +if __name__ == '__main__': + SendallTest().main() diff --git a/test/functional/wallet_signer.py b/test/functional/wallet_signer.py index 423cfecdc0..8e4e1f5d36 100755 --- a/test/functional/wallet_signer.py +++ b/test/functional/wallet_signer.py @@ -194,6 +194,12 @@ class WalletSignerTest(BitcoinTestFramework): assert(res["complete"]) assert_equal(res["hex"], mock_tx) + self.log.info('Test sendall using hww1') + + res = hww.sendall(recipients=[{dest:0.5}, hww.getrawchangeaddress()],options={"add_to_wallet": False}) + assert(res["complete"]) + assert_equal(res["hex"], mock_tx) + # # Handle error thrown by script # self.set_mock_result(self.nodes[4], "2") # assert_raises_rpc_error(-1, 'Unable to parse JSON', diff --git a/test/lint/lint-all.sh b/test/lint/lint-all.sh index fabc24c91b..fa37fa51c6 100755 --- a/test/lint/lint-all.sh +++ b/test/lint/lint-all.sh @@ -4,7 +4,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # -# This script runs all contrib/devtools/lint-*.sh files, and fails if any exit +# This script runs all contrib/devtools/lint-* files, and fails if any exit # with a non-zero status code. # This script is intentionally locale dependent by not setting "export LC_ALL=C" @@ -18,7 +18,7 @@ LINTALL=$(basename "${BASH_SOURCE[0]}") EXIT_CODE=0 -for f in "${SCRIPTDIR}"/lint-*.sh; do +for f in "${SCRIPTDIR}"/lint-*; do if [ "$(basename "$f")" != "$LINTALL" ]; then if ! "$f"; then echo "^---- failure generated from $f" diff --git a/test/lint/lint-files.py b/test/lint/lint-files.py index 68b795eef7..3723e0ee6a 100755 --- a/test/lint/lint-files.py +++ b/test/lint/lint-files.py @@ -13,6 +13,7 @@ import sys from subprocess import check_output from typing import Optional, NoReturn +CMD_TOP_LEVEL = ["git", "rev-parse", "--show-toplevel"] CMD_ALL_FILES = "git ls-files -z --full-name" CMD_SOURCE_FILES = 'git ls-files -z --full-name -- "*.[cC][pP][pP]" "*.[hH]" "*.[pP][yY]" "*.[sS][hH]"' CMD_SHEBANG_FILES = "git grep --full-name --line-number -I '^#!'" @@ -184,6 +185,8 @@ def check_shebang_file_permissions() -> int: def main() -> NoReturn: + root_dir = check_output(CMD_TOP_LEVEL).decode("utf8").strip() + os.chdir(root_dir) failed_tests = 0 failed_tests += check_all_filenames() failed_tests += check_source_filenames() diff --git a/test/lint/lint-files.sh b/test/lint/lint-files.sh deleted file mode 100755 index 86d7fc724a..0000000000 --- a/test/lint/lint-files.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C - -set -e -cd "$(dirname "$0")/../.." -test/lint/lint-files.py diff --git a/test/lint/lint-format-strings.sh b/test/lint/lint-format-strings.sh index d98f12b1a1..73730f16b3 100755 --- a/test/lint/lint-format-strings.sh +++ b/test/lint/lint-format-strings.sh @@ -29,7 +29,7 @@ FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS=( ) EXIT_CODE=0 -if ! python3 -m doctest test/lint/lint-format-strings.py; then +if ! python3 -m doctest "test/lint/run-lint-format-strings.py"; then EXIT_CODE=1 fi for S in "${FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS[@]}"; do @@ -37,7 +37,7 @@ for S in "${FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS[@]}"; do for MATCHING_FILE in $(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(leveldb|secp256k1|minisketch|tinyformat|univalue|test/fuzz/strprintf.cpp)"); do MATCHING_FILES+=("${MATCHING_FILE}") done - if ! test/lint/lint-format-strings.py --skip-arguments "${SKIP_ARGUMENTS}" "${FUNCTION_NAME}" "${MATCHING_FILES[@]}"; then + if ! "test/lint/run-lint-format-strings.py" --skip-arguments "${SKIP_ARGUMENTS}" "${FUNCTION_NAME}" "${MATCHING_FILES[@]}"; then EXIT_CODE=1 fi done diff --git a/test/lint/lint-spelling.sh b/test/lint/lint-spelling.sh index b3e558b02a..8808fbc3c6 100755 --- a/test/lint/lint-spelling.sh +++ b/test/lint/lint-spelling.sh @@ -14,7 +14,7 @@ if ! command -v codespell > /dev/null; then exit 0 fi -IGNORE_WORDS_FILE=test/lint/lint-spelling.ignore-words.txt +IGNORE_WORDS_FILE="test/lint/spelling.ignore-words.txt" mapfile -t FILES < <(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/qt/locale/" ":(exclude)src/qt/*.qrc" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" ":(exclude)contrib/builder-keys/keys.txt" ":(exclude)contrib/guix/patches") if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} "${FILES[@]}"; then echo "^ Warning: codespell identified likely spelling errors. Any false positives? Add them to the list of ignored words in ${IGNORE_WORDS_FILE}" diff --git a/test/lint/lint-format-strings.py b/test/lint/run-lint-format-strings.py index b814446125..b814446125 100755 --- a/test/lint/lint-format-strings.py +++ b/test/lint/run-lint-format-strings.py diff --git a/test/lint/lint-spelling.ignore-words.txt b/test/lint/spelling.ignore-words.txt index afdb0692d8..afdb0692d8 100644 --- a/test/lint/lint-spelling.ignore-words.txt +++ b/test/lint/spelling.ignore-words.txt |