diff options
321 files changed, 5927 insertions, 1852 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 3c59e41a13..70fa76705e 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -43,7 +43,6 @@ env: # Global defaults # The following specific types should exist, with the following requirements: # - small: For an x86_64 machine, recommended to have 2 CPUs and 8 GB of memory. # - medium: For an x86_64 machine, recommended to have 4 CPUs and 16 GB of memory. -# - noble: For a machine running the Linux kernel shipped with exaclty Ubuntu Noble 24.04. The machine is recommended to have 4 CPUs and 16 GB of memory. # - arm64: For an aarch64 machine, recommended to have 2 CPUs and 8 GB of memory. # https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks @@ -160,19 +159,6 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" task: - name: 'ASan + LSan + UBSan + integer, no depends, USDT' - enable_bpfcc_script: - # In the image build step, no external environment variables are available, - # so any settings will need to be written to the settings env file: - - sed -i "s|\${CIRRUS_CI}|true|g" ./ci/test/00_setup_env_native_asan.sh - << : *GLOBAL_TASK_TEMPLATE - persistent_worker: - labels: - type: noble # Must use this specific worker (needed for USDT functional tests) - env: - FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" - -task: name: 'fuzzer,address,undefined,integer, no depends' << : *GLOBAL_TASK_TEMPLATE persistent_worker: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5c322fc23..ab314c47b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,6 @@ concurrency: cancel-in-progress: true env: - DANGER_RUN_CI_ON_HOST: 1 CI_FAILFAST_TEST_LEAVE_DANGLING: 1 # GHA does not care about dangling processes and setting this variable avoids killing the CI script itself on error MAKEJOBS: '-j10' @@ -81,6 +80,7 @@ jobs: timeout-minutes: 120 env: + DANGER_RUN_CI_ON_HOST: 1 FILE_ENV: './ci/test/00_setup_env_mac_native.sh' BASE_ROOT_DIR: ${{ github.workspace }} @@ -153,12 +153,10 @@ jobs: - name: Get tool information run: | - msbuild -version | Out-File -FilePath "$env:GITHUB_WORKSPACE\msbuild_version" - Get-Content -Path "$env:GITHUB_WORKSPACE\msbuild_version" - $env:VCToolsVersion | Out-File -FilePath "$env:GITHUB_WORKSPACE\toolset_version" - Write-Host "VCToolsVersion $(Get-Content -Path "$env:GITHUB_WORKSPACE\toolset_version")" - $env:CI_QT_URL | Out-File -FilePath "$env:GITHUB_WORKSPACE\qt_url" - $env:CI_QT_CONF | Out-File -FilePath "$env:GITHUB_WORKSPACE\qt_conf" + msbuild -version | Tee-Object -FilePath "msbuild_version" + $env:VCToolsVersion | Tee-Object -FilePath "toolset_version" + $env:CI_QT_URL | Out-File -FilePath "qt_url" + $env:CI_QT_CONF | Out-File -FilePath "qt_conf" py -3 --version Write-Host "PowerShell version $($PSVersionTable.PSVersion.ToString())" @@ -241,10 +239,8 @@ jobs: run: | Set-Location "$env:VCPKG_INSTALLATION_ROOT" Add-Content -Path "triplets\x64-windows-static.cmake" -Value "set(VCPKG_BUILD_TYPE release)" - Add-Content -Path "triplets\x64-windows-static.cmake" -Value "set(VCPKG_PLATFORM_TOOLSET_VERSION $env:VCToolsVersion)" .\vcpkg.exe --vcpkg-root "$env:VCPKG_INSTALLATION_ROOT" integrate install - git rev-parse HEAD | Out-File -FilePath "$env:GITHUB_WORKSPACE\vcpkg_commit" - Get-Content -Path "$env:GITHUB_WORKSPACE\vcpkg_commit" + git rev-parse HEAD | Tee-Object -FilePath "$env:GITHUB_WORKSPACE\vcpkg_commit" - name: vcpkg tools cache uses: actions/cache@v4 @@ -308,3 +304,42 @@ jobs: BITCOINFUZZ: "${{ github.workspace}}\\src\\fuzz.exe" shell: cmd run: py -3 test\fuzz\test_runner.py --par %NUMBER_OF_PROCESSORS% --loglevel DEBUG %RUNNER_TEMP%\qa-assets\fuzz_seed_corpus + + asan-lsan-ubsan-integer-no-depends-usdt: + name: 'ASan + LSan + UBSan + integer, no depends, USDT' + runs-on: ubuntu-24.04 # has to match container in ci/test/00_setup_env_native_asan.sh for tracing tools + # No need to run on the read-only mirror, unless it is a PR. + if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request' + timeout-minutes: 120 + env: + FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set Ccache directory + run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV" + + - name: Restore Ccache cache + id: ccache-cache + uses: actions/cache/restore@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ${{ github.job }}-ccache-${{ github.run_id }} + restore-keys: ${{ github.job }}-ccache- + + - name: Enable bpfcc script + # In the image build step, no external environment variables are available, + # so any settings will need to be written to the settings env file: + run: sed -i "s|\${INSTALL_BCC_TRACING_TOOLS}|true|g" ./ci/test/00_setup_env_native_asan.sh + + - name: CI script + run: ./ci/test_run_all.sh + + - name: Save Ccache cache + uses: actions/cache/save@v4 + if: github.event_name != 'pull_request' && steps.ccache-cache.outputs.cache-hit != 'true' + with: + path: ${{ env.CCACHE_DIR }} + # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache + key: ${{ github.job }}-ccache-${{ github.run_id }} diff --git a/build-aux/m4/ax_cxx_compile_stdcxx.m4 b/build-aux/m4/ax_cxx_compile_stdcxx.m4 index 8a2df5627f..51a35054d0 100644 --- a/build-aux/m4/ax_cxx_compile_stdcxx.m4 +++ b/build-aux/m4/ax_cxx_compile_stdcxx.m4 @@ -983,7 +983,7 @@ m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_20], [[ #error "This is not a C++ compiler" -#elif __cplusplus < 201709L // Temporary patch on top of upstream to allow g++-10 +#elif __cplusplus < 202002L #error "This is not a C++20 compiler" diff --git a/ci/test/00_setup_env_mac_cross.sh b/ci/test/00_setup_env_mac_cross.sh index 31c4bff6ae..f607c93ae6 100755 --- a/ci/test/00_setup_env_mac_cross.sh +++ b/ci/test/00_setup_env_mac_cross.sh @@ -9,9 +9,9 @@ export LC_ALL=C.UTF-8 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} export CONTAINER_NAME=ci_macos_cross -export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04" +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export HOST=x86_64-apple-darwin -export PACKAGES="zip" +export PACKAGES="clang lld llvm zip" export XCODE_VERSION=15.0 export XCODE_BUILD_ID=15A240d export RUN_UNIT_TESTS=false diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 668e9ecc8a..0fd169f523 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -7,8 +7,10 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" -# Only install BCC tracing packages in Cirrus CI. -if [[ "${CIRRUS_CI}" == "true" ]]; then + +# Only install BCC tracing packages in CI. Container has to match the host for BCC to work. +if [[ "${INSTALL_BCC_TRACING_TOOLS}" == "true" ]]; then + # Required for USDT functional tests to run BPFCC_PACKAGE="bpfcc-tools linux-headers-$(uname --kernel-release)" export CI_CONTAINER_CAP="--privileged -v /sys/kernel:/sys/kernel:rw" else diff --git a/ci/test/02_run_container.sh b/ci/test/02_run_container.sh index 86bb856d17..17a940de07 100755 --- a/ci/test/02_run_container.sh +++ b/ci/test/02_run_container.sh @@ -16,6 +16,7 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then # System-dependent env vars must be kept as is. So read them from the container. docker run --rm "${CI_IMAGE_NAME_TAG}" bash -c "env | grep --extended-regexp '^(HOME|PATH|USER)='" | tee --append "/tmp/env-$USER-$CONTAINER_NAME" echo "Creating $CI_IMAGE_NAME_TAG container to run in" + DOCKER_BUILDKIT=1 docker build \ --file "${BASE_READ_ONLY_DIR}/ci/test_imagefile" \ --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \ @@ -23,11 +24,14 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then --label="${CI_IMAGE_LABEL}" \ --tag="${CONTAINER_NAME}" \ "${BASE_READ_ONLY_DIR}" + docker volume create "${CONTAINER_NAME}_ccache" || true docker volume create "${CONTAINER_NAME}_depends" || true docker volume create "${CONTAINER_NAME}_depends_sources" || true docker volume create "${CONTAINER_NAME}_previous_releases" || true + docker network create --ipv6 --subnet 1111:1111::/112 ci-ip6net || true + if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then echo "Restart docker before run to stop and clear all containers started with --rm" podman container rm --force --all # Similar to "systemctl restart docker" @@ -54,6 +58,7 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then --mount "type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR" \ --env-file /tmp/env-$USER-$CONTAINER_NAME \ --name "$CONTAINER_NAME" \ + --network ci-ip6net \ "$CONTAINER_NAME") export CI_CONTAINER_ID export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}" diff --git a/configure.ac b/configure.ac index 6af2c5aba8..ab369cc98a 100644 --- a/configure.ac +++ b/configure.ac @@ -206,9 +206,9 @@ AC_ARG_WITH([qrencode], AC_ARG_ENABLE([hardening], [AS_HELP_STRING([--disable-hardening], - [do not attempt to harden the resulting executables (default is to harden when possible)])], + [do not attempt to harden the resulting executables (default is to harden)])], [use_hardening=$enableval], - [use_hardening=auto]) + [use_hardening=yes]) AC_ARG_ENABLE([reduce-exports], [AS_HELP_STRING([--enable-reduce-exports], @@ -238,12 +238,6 @@ AC_ARG_ENABLE([lcov], [use_lcov=$enableval], [use_lcov=no]) -AC_ARG_ENABLE([lcov-branch-coverage], - [AS_HELP_STRING([--enable-lcov-branch-coverage], - [enable lcov testing branch coverage (default is no)])], - [use_lcov_branch=yes], - [use_lcov_branch=no]) - AC_ARG_ENABLE([zmq], [AS_HELP_STRING([--disable-zmq], [disable ZMQ notifications])], @@ -287,13 +281,6 @@ AC_ARG_WITH([sanitizers], [comma separated list of extra sanitizers to build with (default is none enabled)])], [use_sanitizers=$withval]) -dnl Enable gprof profiling -AC_ARG_ENABLE([gprof], - [AS_HELP_STRING([--enable-gprof], - [use gprof profiling compiler flags (default is no)])], - [enable_gprof=$enableval], - [enable_gprof=no]) - dnl Turn warnings into errors AC_ARG_ENABLE([werror], [AS_HELP_STRING([--enable-werror], @@ -417,15 +404,12 @@ AX_CHECK_COMPILE_FLAG([-Wsuggest-override], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsug AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wimplicit-fallthrough"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wunreachable-code], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunreachable-code"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wdocumentation], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdocumentation"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wself-assign], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wself-assign"], [], [$CXXFLAG_WERROR]) dnl Some compilers (gcc) ignore unknown -Wno-* options, but warn about all dnl unknown options if any other warning is produced. Test the -Wfoo case, and dnl set the -Wno-foo case if it works. AX_CHECK_COMPILE_FLAG([-Wunused-parameter], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-parameter"], [], [$CXXFLAG_WERROR]) -AX_CHECK_COMPILE_FLAG([-Wself-assign], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-self-assign"], [], [$CXXFLAG_WERROR]) -if test "$suppress_external_warnings" != "yes" ; then - AX_CHECK_COMPILE_FLAG([-Wdeprecated-copy], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-deprecated-copy"], [], [$CXXFLAG_WERROR]) -fi dnl Don't allow extended (non-ASCII) symbols in identifiers. This is easier for code review. AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers], [CORE_CXXFLAGS="$CORE_CXXFLAGS -fno-extended-identifiers"], [], [$CXXFLAG_WERROR]) @@ -821,10 +805,8 @@ if test "$use_lcov" = "yes"; then AX_CHECK_COMPILE_FLAG([--coverage],[CORE_CXXFLAGS="$CORE_CXXFLAGS --coverage"], [AC_MSG_ERROR([lcov testing requested but --coverage flag does not work])]) CORE_CXXFLAGS="$CORE_CXXFLAGS -Og" -fi -if test "$use_lcov_branch" != "no"; then - AC_SUBST(LCOV_OPTS, "$LCOV_OPTS --rc lcov_branch_coverage=1") + AC_SUBST(LCOV_OPTS) fi dnl Check for endianness @@ -855,30 +837,12 @@ if test "$ac_cv_sys_large_files" != "" && CORE_CPPFLAGS="$CORE_CPPFLAGS -D_LARGE_FILES=$ac_cv_sys_large_files" fi -if test "$enable_gprof" = "yes"; then - dnl -pg is incompatible with -pie. Since hardening and profiling together doesn't make sense, - dnl we simply make them mutually exclusive here. Additionally, hardened toolchains may force - dnl -pie by default, in which case it needs to be turned off with -no-pie. - - if test "$use_hardening" = "yes"; then - AC_MSG_ERROR([gprof profiling is not compatible with hardening. Reconfigure with --disable-hardening or --disable-gprof]) - fi - use_hardening=no - AX_CHECK_COMPILE_FLAG([-pg],[GPROF_CXXFLAGS="-pg"], - [AC_MSG_ERROR([gprof profiling requested but not available])], [$CXXFLAG_WERROR]) - - AX_CHECK_LINK_FLAG([-no-pie], [GPROF_LDFLAGS="-no-pie"]) - AX_CHECK_LINK_FLAG([-pg], [GPROF_LDFLAGS="$GPROF_LDFLAGS -pg"], - [AC_MSG_ERROR([gprof profiling requested but not available])], [$GPROF_LDFLAGS]) -fi - if test "$TARGET_OS" != "windows"; then dnl All windows code is PIC, forcing it on just adds useless compile warnings AX_CHECK_COMPILE_FLAG([-fPIC], [PIC_FLAGS="-fPIC"]) fi if test "$use_hardening" != "no"; then - use_hardening=yes AX_CHECK_COMPILE_FLAG([-Wstack-protector], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -Wstack-protector"]) AX_CHECK_COMPILE_FLAG([-fstack-protector-all], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-protector-all"]) @@ -1695,8 +1659,6 @@ AC_SUBST(WARN_CXXFLAGS) AC_SUBST(NOWARN_CXXFLAGS) AC_SUBST(DEBUG_CXXFLAGS) AC_SUBST(ERROR_CXXFLAGS) -AC_SUBST(GPROF_CXXFLAGS) -AC_SUBST(GPROF_LDFLAGS) AC_SUBST(HARDENED_CXXFLAGS) AC_SUBST(HARDENED_CPPFLAGS) AC_SUBST(HARDENED_LDFLAGS) @@ -1804,7 +1766,6 @@ echo " with natpmp = $use_natpmp" echo " USDT tracing = $use_usdt" echo " sanitizers = $use_sanitizers" echo " debug enabled = $enable_debug" -echo " gprof enabled = $enable_gprof" echo " werror = $enable_werror" echo echo " target os = $host_os" @@ -1814,8 +1775,8 @@ echo " CC = $CC" echo " CFLAGS = $PTHREAD_CFLAGS $SANITIZER_CFLAGS $CFLAGS" echo " CPPFLAGS = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CORE_CPPFLAGS $CPPFLAGS" echo " CXX = $CXX" -echo " CXXFLAGS = $CORE_CXXFLAGS $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $SANITIZER_CXXFLAGS $CXXFLAGS" -echo " LDFLAGS = $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $SANITIZER_LDFLAGS $CORE_LDFLAGS $LDFLAGS" +echo " CXXFLAGS = $CORE_CXXFLAGS $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $SANITIZER_CXXFLAGS $CXXFLAGS" +echo " LDFLAGS = $PTHREAD_LIBS $HARDENED_LDFLAGS $SANITIZER_LDFLAGS $CORE_LDFLAGS $LDFLAGS" echo " AR = $AR" echo " ARFLAGS = $ARFLAGS" echo diff --git a/contrib/devtools/check-deps.sh b/contrib/devtools/check-deps.sh new file mode 100755 index 0000000000..9d2eebe14d --- /dev/null +++ b/contrib/devtools/check-deps.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash + +export LC_ALL=C +set -Eeuo pipefail + +# Declare paths to libraries +declare -A LIBS +LIBS[cli]="libbitcoin_cli.a" +LIBS[common]="libbitcoin_common.a" +LIBS[consensus]="libbitcoin_consensus.a" +LIBS[crypto]="crypto/.libs/libbitcoin_crypto_base.a crypto/.libs/libbitcoin_crypto_x86_shani.a crypto/.libs/libbitcoin_crypto_sse41.a crypto/.libs/libbitcoin_crypto_avx2.a" +LIBS[node]="libbitcoin_node.a" +LIBS[util]="libbitcoin_util.a" +LIBS[wallet]="libbitcoin_wallet.a" +LIBS[wallet_tool]="libbitcoin_wallet_tool.a" + +# Declare allowed dependencies "X Y" where X is allowed to depend on Y. This +# list is taken from doc/design/libraries.md. +ALLOWED_DEPENDENCIES=( + "cli common" + "cli util" + "common consensus" + "common crypto" + "common util" + "consensus crypto" + "node common" + "node consensus" + "node crypto" + "node kernel" + "node util" + "util crypto" + "wallet common" + "wallet crypto" + "wallet util" + "wallet_tool util" + "wallet_tool wallet" +) + +# Add minor dependencies omitted from doc/design/libraries.md to keep the +# dependency diagram simple. +ALLOWED_DEPENDENCIES+=( + "wallet consensus" + "wallet_tool common" + "wallet_tool crypto" +) + +# Declare list of known errors that should be suppressed. +declare -A SUPPRESS +# init.cpp file currently calls Berkeley DB sanity check function on startup, so +# there is an undocumented dependency of the node library on the wallet library. +SUPPRESS["libbitcoin_node_a-init.o libbitcoin_wallet_a-bdb.o _ZN6wallet27BerkeleyDatabaseSanityCheckEv"]=1 +# init/common.cpp file calls InitError and InitWarning from interface_ui which +# is currently part of the node library. interface_ui should just be part of the +# common library instead, and is moved in +# https://github.com/bitcoin/bitcoin/issues/10102 +SUPPRESS["libbitcoin_common_a-common.o libbitcoin_node_a-interface_ui.o _Z11InitWarningRK13bilingual_str"]=1 +SUPPRESS["libbitcoin_common_a-common.o libbitcoin_node_a-interface_ui.o _Z9InitErrorRK13bilingual_str"]=1 +# rpc/external_signer.cpp adds defines node RPC methods but is built as part of the +# common library. It should be moved to the node library instead. +SUPPRESS["libbitcoin_common_a-external_signer.o libbitcoin_node_a-server.o _ZN9CRPCTable13appendCommandERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPK11CRPCCommand"]=1 + +usage() { + echo "Usage: $(basename "${BASH_SOURCE[0]}") [BUILD_DIR]" +} + +# Output makefile targets, converting library .a paths to libtool .la targets +lib_targets() { + for lib in "${!LIBS[@]}"; do + for lib_path in ${LIBS[$lib]}; do + # shellcheck disable=SC2001 + sed 's:/.libs/\(.*\)\.a$:/\1.la:g' <<<"$lib_path" + done + done +} + +# Extract symbol names and object names and write to text files +extract_symbols() { + local temp_dir="$1" + for lib in "${!LIBS[@]}"; do + for lib_path in ${LIBS[$lib]}; do + nm -o "$lib_path" | grep ' T ' | awk '{print $3, $1}' >> "${temp_dir}/${lib}_exports.txt" + nm -o "$lib_path" | grep ' U ' | awk '{print $3, $1}' >> "${temp_dir}/${lib}_imports.txt" + awk '{print $1}' "${temp_dir}/${lib}_exports.txt" | sort -u > "${temp_dir}/${lib}_exported_symbols.txt" + awk '{print $1}' "${temp_dir}/${lib}_imports.txt" | sort -u > "${temp_dir}/${lib}_imported_symbols.txt" + done + done +} + +# Lookup object name(s) corresponding to symbol name in text file +obj_names() { + local symbol="$1" + local txt_file="$2" + sed -n "s/^$symbol [^:]\\+:\\([^:]\\+\\):[^:]*\$/\\1/p" "$txt_file" | sort -u +} + +# Iterate through libraries and find disallowed dependencies +check_libraries() { + local temp_dir="$1" + local result=0 + for src in "${!LIBS[@]}"; do + for dst in "${!LIBS[@]}"; do + if [ "$src" != "$dst" ] && ! is_allowed "$src" "$dst"; then + if ! check_disallowed "$src" "$dst" "$temp_dir"; then + result=1 + fi + fi + done + done + check_not_suppressed + return $result +} + +# Return whether src library is allowed to depend on dst. +is_allowed() { + local src="$1" + local dst="$2" + for allowed in "${ALLOWED_DEPENDENCIES[@]}"; do + if [ "$src $dst" = "$allowed" ]; then + return 0 + fi + done + return 1 +} + +# Return whether src library imports any symbols from dst, assuming src is not +# allowed to depend on dst. +check_disallowed() { + local src="$1" + local dst="$2" + local temp_dir="$3" + local result=0 + + # Loop over symbol names exported by dst and imported by src + while read symbol; do + local dst_obj + dst_obj=$(obj_names "$symbol" "${temp_dir}/${dst}_exports.txt") + while read src_obj; do + if ! check_suppress "$src_obj" "$dst_obj" "$symbol"; then + echo "Error: $src_obj depends on $dst_obj symbol '$(c++filt "$symbol")', can suppess with:" + echo " SUPPRESS[\"$src_obj $dst_obj $symbol\"]=1" + result=1 + fi + done < <(obj_names "$symbol" "${temp_dir}/${src}_imports.txt") + done < <(comm -12 "${temp_dir}/${dst}_exported_symbols.txt" "${temp_dir}/${src}_imported_symbols.txt") + return $result +} + +# Declare array to track errors which were suppressed. +declare -A SUPPRESSED + +# Return whether error should be suppressed and record suppresssion in +# SUPPRESSED array. +check_suppress() { + local src_obj="$1" + local dst_obj="$2" + local symbol="$3" + for suppress in "${!SUPPRESS[@]}"; do + read suppress_src suppress_dst suppress_pattern <<<"$suppress" + if [[ "$src_obj" == "$suppress_src" && "$dst_obj" == "$suppress_dst" && "$symbol" =~ $suppress_pattern ]]; then + SUPPRESSED["$suppress"]=1 + return 0 + fi + done + return 1 +} + +# Warn about error which were supposed to be suppress, but were not encountered. +check_not_suppressed() { + for suppress in "${!SUPPRESS[@]}"; do + if [[ ! -v SUPPRESSED[$suppress] ]]; then + echo >&2 "Warning: suppression '$suppress' was ignored, consider deleting." + fi + done +} + +# Check arguments. +if [ "$#" = 0 ]; then + BUILD_DIR="$(dirname "${BASH_SOURCE[0]}")/../../src" +elif [ "$#" = 1 ]; then + BUILD_DIR="$1" +else + echo >&2 "Error: wrong number of arguments." + usage >&2 + exit 1 +fi +if [ ! -f "$BUILD_DIR/Makefile" ]; then + echo >&2 "Error: directory '$BUILD_DIR' does not contain a makefile, please specify path to build directory for library targets." + usage >&2 + exit 1 +fi + +# Build libraries and run checks. +cd "$BUILD_DIR" +# shellcheck disable=SC2046 +make -j"$(nproc)" $(lib_targets) +TEMP_DIR="$(mktemp -d)" +extract_symbols "$TEMP_DIR" +if check_libraries "$TEMP_DIR"; then + echo "Success! No unexpected dependencies were detected." +else + echo >&2 "Error: Unexpected dependencies were detected. Check previous output." +fi +rm -r "$TEMP_DIR" diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index e4a62c2072..6613874ce3 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -239,8 +239,8 @@ def check_MACHO_sdk(binary) -> bool: return True return False -def check_MACHO_ld64(binary) -> bool: - if binary.build_version.tools[0].version == [17, 0, 6]: +def check_MACHO_lld(binary) -> bool: + if binary.build_version.tools[0].version == [18, 1, 6]: return True return False @@ -282,7 +282,7 @@ lief.EXE_FORMATS.MACHO: [ ('DYNAMIC_LIBRARIES', check_MACHO_libraries), ('MIN_OS', check_MACHO_min_os), ('SDK', check_MACHO_sdk), - ('LD64', check_MACHO_ld64), + ('LLD', check_MACHO_lld), ], lief.EXE_FORMATS.PE: [ ('DYNAMIC_LIBRARIES', check_PE_libraries), diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index 870938cb52..2ea574fe4b 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -361,6 +361,10 @@ INFO: Building ${VERSION:?not set} for platform triple ${HOST:?not set}: ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' ...outputting in: '$(outdir_for_host "$HOST")' ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")' + ADDITIONAL FLAGS (if set) + ADDITIONAL_GUIX_COMMON_FLAGS: ${ADDITIONAL_GUIX_COMMON_FLAGS} + ADDITIONAL_GUIX_ENVIRONMENT_FLAGS: ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} + ADDITIONAL_GUIX_TIMEMACHINE_FLAGS: ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} EOF # Run the build script 'contrib/guix/libexec/build.sh' in the build diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index f589ac7a55..9bc8c0e75d 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -178,8 +178,7 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \ x86_64_linux_AR=x86_64-linux-gnu-gcc-ar \ x86_64_linux_RANLIB=x86_64-linux-gnu-gcc-ranlib \ x86_64_linux_NM=x86_64-linux-gnu-gcc-nm \ - x86_64_linux_STRIP=x86_64-linux-gnu-strip \ - FORCE_USE_SYSTEM_CLANG=1 + x86_64_linux_STRIP=x86_64-linux-gnu-strip ########################### diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index ce6a9562b4..80bfb2875f 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -51,7 +51,7 @@ fi time-machine() { # shellcheck disable=SC2086 guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=dc4842797bfdc5f9f3f5f725bf189c2b68bd6b5a \ + --commit=f0bb724211872cd6158fce6162e0b8c73efed126 \ --cores="$JOBS" \ --keep-failed \ --fallback \ diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 40500ccb88..53569d7f7d 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -532,9 +532,9 @@ inspecting signatures in Mach-O binaries.") ((string-contains target "darwin") (list ;; Native GCC 11 toolchain gcc-toolchain-11 - clang-toolchain-17 - lld-17 - (make-lld-wrapper lld-17 #:lld-as-ld? #t) + clang-toolchain-18 + lld-18 + (make-lld-wrapper lld-18 #:lld-as-ld? #t) python-signapple zip)) (else '()))))) diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 78f61685e1..d47ee6774e 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -56,30 +56,22 @@ The `sha256sum` should be `c0c2e7bb92c1fee0c4e9f3a485e4530786732d6c6dd9e9f418c28 ## Deterministic macOS App Notes -macOS Applications are created in Linux using a recent LLVM. +macOS Applications are created on Linux using a recent LLVM. -Apple uses `clang` extensively for development and has upstreamed the necessary -functionality so that a vanilla clang can take advantage. It supports the use of `-F`, -`-target`, `-mmacosx-version-min`, and `-isysroot`, which are all necessary when -building for macOS. +All builds must target an Apple SDK. These SDKs are free to download, but not redistributable. +See the SDK Extraction notes above for how to obtain it. -To complicate things further, all builds must target an Apple SDK. These SDKs are free to -download, but not redistributable. See the SDK Extraction notes above for how to obtain it. +The Guix build process has been designed to avoid including the SDK's files in Guix's outputs. +All interim tarballs are fully deterministic and may be freely redistributed. -The Guix process builds 2 sets of files: Linux tools, then Apple binaries which are -created using these tools. The build process has been designed to avoid including the -SDK's files in Guix's outputs. All interim tarballs are fully deterministic and may be freely -redistributed. - -As of OS X 10.9 Mavericks, using an Apple-blessed key to sign binaries is a requirement in -order to satisfy the new Gatekeeper requirements. Because this private key cannot be -shared, we'll have to be a bit creative in order for the build process to remain somewhat -deterministic. Here's how it works: +Using an Apple-blessed key to sign binaries is a requirement to produce (distributable) macOS +binaries. Because this private key cannot be shared, we'll have to be a bit creative in order +for the build process to remain somewhat deterministic. Here's how it works: - Builders use Guix to create an unsigned release. This outputs an unsigned ZIP which - users may choose to bless and run. It also outputs an unsigned app structure in the form - of a tarball. + users may choose to bless, self-codesign, and run. It also outputs an unsigned app structure + in the form of a tarball. - The Apple keyholder uses this unsigned app to create a detached signature, using the - script that is also included there. Detached signatures are available from this [repository](https://github.com/bitcoin-core/bitcoin-detached-sigs). -- Builders feed the unsigned app + detached signature back into Guix. It uses the - pre-built tools to recombine the pieces into a deterministic ZIP. + included script. Detached signatures are available from this [repository](https://github.com/bitcoin-core/bitcoin-detached-sigs). +- Builders feed the unsigned app + detached signature back into Guix, which combines the + pieces into a deterministic ZIP. diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk index b73f5cba14..86a6262b5c 100755 --- a/contrib/macdeploy/gen-sdk +++ b/contrib/macdeploy/gen-sdk @@ -8,21 +8,6 @@ import gzip import os import contextlib -# monkey-patch Python 3.8 and older to fix wrong TAR header handling -# see https://github.com/bitcoin/bitcoin/pull/24534 -# and https://github.com/python/cpython/pull/18080 for more info -if sys.version_info < (3, 9): - _old_create_header = tarfile.TarInfo._create_header - def _create_header(info, format, encoding, errors): - buf = _old_create_header(info, format, encoding, errors) - # replace devmajor/devminor with binary zeroes - buf = buf[:329] + bytes(16) + buf[345:] - # recompute checksum - chksum = tarfile.calc_chksums(buf)[0] - buf = buf[:-364] + bytes("%06o\0" % chksum, "ascii") + buf[-357:] - return buf - tarfile.TarInfo._create_header = staticmethod(_create_header) - @contextlib.contextmanager def cd(path): """Context manager that restores PWD even if an exception was raised.""" diff --git a/contrib/seeds/generate-seeds.py b/contrib/seeds/generate-seeds.py index e921757802..f67e7b0f4c 100755 --- a/contrib/seeds/generate-seeds.py +++ b/contrib/seeds/generate-seeds.py @@ -29,7 +29,6 @@ These should be pasted into `src/chainparamsseeds.h`. from base64 import b32decode from enum import Enum -import struct import sys import os import re @@ -115,13 +114,13 @@ def parse_spec(s): def ser_compact_size(l): r = b"" if l < 253: - r = struct.pack("B", l) + r = l.to_bytes(1, "little") elif l < 0x10000: - r = struct.pack("<BH", 253, l) + r = (253).to_bytes(1, "little") + l.to_bytes(2, "little") elif l < 0x100000000: - r = struct.pack("<BI", 254, l) + r = (254).to_bytes(1, "little") + l.to_bytes(4, "little") else: - r = struct.pack("<BQ", 255, l) + r = (255).to_bytes(1, "little") + l.to_bytes(8, "little") return r def bip155_serialize(spec): @@ -129,10 +128,10 @@ def bip155_serialize(spec): Serialize (networkID, addr, port) tuple to BIP155 binary format. ''' r = b"" - r += struct.pack('B', spec[0].value) + r += spec[0].value.to_bytes(1, "little") r += ser_compact_size(len(spec[1])) r += spec[1] - r += struct.pack('>H', spec[2]) + r += spec[2].to_bytes(2, "big") return r def process_nodes(g, f, structname): diff --git a/contrib/signet/miner b/contrib/signet/miner index e5daf9f993..4216ada5fa 100755 --- a/contrib/signet/miner +++ b/contrib/signet/miner @@ -52,20 +52,20 @@ def signet_txs(block, challenge): mroot = block.get_merkle_root(hashes) sd = b"" - sd += struct.pack("<i", block.nVersion) + sd += block.nVersion.to_bytes(4, "little", signed=True) sd += ser_uint256(block.hashPrevBlock) sd += ser_uint256(mroot) - sd += struct.pack("<I", block.nTime) + sd += block.nTime.to_bytes(4, "little") to_spend = CTransaction() - to_spend.nVersion = 0 + to_spend.version = 0 to_spend.nLockTime = 0 to_spend.vin = [CTxIn(COutPoint(0, 0xFFFFFFFF), b"\x00" + CScriptOp.encode_op_pushdata(sd), 0)] to_spend.vout = [CTxOut(0, challenge)] to_spend.rehash() spend = CTransaction() - spend.nVersion = 0 + spend.version = 0 spend.nLockTime = 0 spend.vin = [CTxIn(COutPoint(to_spend.sha256, 0), b"", 0)] spend.vout = [CTxOut(0, b"\x6a")] diff --git a/depends/Makefile b/depends/Makefile index fee426d8db..52a9a14e56 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -181,8 +181,6 @@ all_packages = $(packages) $(native_packages) meta_depends = Makefile config.guess config.sub funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk -$(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain) - include funcs.mk final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in) diff --git a/depends/README.md b/depends/README.md index 17def65d3e..adaeed468f 100644 --- a/depends/README.md +++ b/depends/README.md @@ -43,13 +43,17 @@ The paths are automatically configured and no other options are needed. ### Install the required dependencies: Ubuntu & Debian +#### Common + + apt install automake bison cmake curl libtool make patch pkg-config python3 xz-utils + #### For macOS cross compilation - sudo apt-get install curl bsdmainutils cmake zip + apt install clang lld llvm g++ zip -Note: You must obtain the macOS SDK before proceeding with a cross-compile. -Under the depends directory, create a subdirectory named `SDKs`. -Then, place the extracted SDK under this new directory. +Clang 18 or later is required. You must also obtain the macOS SDK before +proceeding with a cross-compile. Under the depends directory, create a +subdirectory named `SDKs`. Then, place the extracted SDK under this new directory. For more information, see [SDK Extraction](../contrib/macdeploy/README.md#sdk-extraction). #### For Win64 cross compilation @@ -60,7 +64,7 @@ For more information, see [SDK Extraction](../contrib/macdeploy/README.md#sdk-ex Common linux dependencies: - sudo apt-get install make automake cmake curl g++-multilib libtool binutils bsdmainutils pkg-config python3 patch bison + sudo apt-get install g++-multilib binutils For linux ARM cross compilation: @@ -115,9 +119,6 @@ The following can be set when running make: `make FOO=bar` - `DEBUG`: Disable some optimizations and enable more runtime checking - `HOST_ID_SALT`: Optional salt to use when generating host package ids - `BUILD_ID_SALT`: Optional salt to use when generating build package ids -- `FORCE_USE_SYSTEM_CLANG`: (EXPERTS ONLY) When cross-compiling for macOS, use Clang found in the - system's `$PATH` rather than the default prebuilt release of Clang - from llvm.org. Clang 8 or later is required - `LOG`: Use file-based logging for individual packages. During a package build its log file resides in the `depends` directory, and the log file is printed out automatically in case of build error. After successful build log files are moved along with package archives diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk index d84c23ed44..2b59353e84 100644 --- a/depends/builders/darwin.mk +++ b/depends/builders/darwin.mk @@ -18,7 +18,6 @@ darwin_STRIP:=$(shell xcrun -f strip) darwin_OBJDUMP:=$(shell xcrun -f objdump) darwin_NM:=$(shell xcrun -f nm) darwin_DSYMUTIL:=$(shell xcrun -f dsymutil) -darwin_native_toolchain= x86_64_darwin_CFLAGS += -arch x86_64 x86_64_darwin_CXXFLAGS += -arch x86_64 diff --git a/depends/funcs.mk b/depends/funcs.mk index 537051c030..3c0dc7a7fc 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -46,7 +46,7 @@ endef define int_get_build_id $(eval $(1)_dependencies += $($(1)_$(host_arch)_$(host_os)_dependencies) $($(1)_$(host_os)_dependencies)) -$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($(1)_dependencies))) +$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($(1)_dependencies))) $(foreach dep,$($(1)_all_dependencies),$(eval $(1)_build_id_deps+=$(dep)-$($(dep)_version)-$($(dep)_recipe_hash))) $(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id)) $(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))) @@ -297,6 +297,3 @@ $(foreach package,$(all_packages),$(eval $(call int_config_attach_build_config,$ #create build targets $(foreach package,$(all_packages),$(eval $(call int_add_cmds,$(package)))) - -#special exception: if a toolchain package exists, all non-native packages depend on it -$(foreach package,$(packages),$(eval $($(package)_extracted): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) )) diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index a64008d6aa..564381d1e9 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -2,39 +2,10 @@ OSX_MIN_VERSION=11.0 OSX_SDK_VERSION=14.0 XCODE_VERSION=15.0 XCODE_BUILD_ID=15A240d -LD64_VERSION=711 +LLD_VERSION=711 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers -ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) -# FORCE_USE_SYSTEM_CLANG is empty, so we use our depends-managed, pinned LLVM -# from llvm.org - -darwin_native_toolchain=native_llvm - -clang_prog=$(build_prefix)/bin/clang -clangxx_prog=$(clang_prog)++ -llvm_config_prog=$(build_prefix)/bin/llvm-config - -llvm_TOOLS=AR NM OBJDUMP RANLIB STRIP - -# Make-only lowercase function -lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1)))))))))))))))))))))))))) - -# For well-known tools provided by LLVM, make sure that their well-known -# variable is set to the full path of the tool, just like how AC_PATH_{TOO,PROG} -# would. -$(foreach TOOL,$(llvm_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/llvm-$(call lc,$(TOOL)))) - -# Clang expects dsymutil to be called dsymutil -darwin_DSYMUTIL=$(build_prefix)/bin/dsymutil - -else -# FORCE_USE_SYSTEM_CLANG is non-empty, so we use the clang from the user's -# system - -darwin_native_toolchain= - # We can't just use $(shell command -v clang) because GNU Make handles builtins # in a special way and doesn't know that `command` is a POSIX-standard builtin # prior to 1af314465e5dfe3e8baa839a32a72e83c04f26ef, first released in v4.2.90. @@ -44,9 +15,6 @@ darwin_native_toolchain= # Source: https://lists.gnu.org/archive/html/bug-make/2017-11/msg00017.html clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang") clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++") -llvm_config_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-config") - -llvm_lib_dir=$(shell $(llvm_config_prog) --libdir) darwin_AR=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-ar") darwin_DSYMUTIL=$(shell $(SHELL) $(.SHELLFLAGS) "command -v dsymutil") @@ -54,7 +22,6 @@ darwin_NM=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-nm") darwin_OBJDUMP=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-objdump") darwin_RANLIB=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-ranlib") darwin_STRIP=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-strip") -endif # Flag explanations: # @@ -63,11 +30,6 @@ endif # Ensures that modern linker features are enabled. See here for more # details: https://github.com/bitcoin/bitcoin/pull/19407. # -# -B$(build_prefix)/bin -# -# Explicitly point to our binaries so that they are -# ensured to be found and preferred over other possibilities. -# # -isysroot$(OSX_SDK) -nostdlibinc # # Disable default include paths built into the compiler as well as @@ -92,7 +54,6 @@ darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ -u LIBRARY_PATH \ $(clang_prog) --target=$(host) \ - -B$(build_prefix)/bin \ -isysroot$(OSX_SDK) -nostdlibinc \ -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks @@ -100,18 +61,17 @@ darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ -u LIBRARY_PATH \ $(clangxx_prog) --target=$(host) \ - -B$(build_prefix)/bin \ -isysroot$(OSX_SDK) -nostdlibinc \ -iwithsysroot/usr/include/c++/v1 \ -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks -darwin_CFLAGS=-pipe -std=$(C_STANDARD) -mmacosx-version-min=$(OSX_MIN_VERSION) -darwin_CXXFLAGS=-pipe -std=$(CXX_STANDARD) -mmacosx-version-min=$(OSX_MIN_VERSION) +darwin_CFLAGS=-pipe -std=$(C_STANDARD) -mmacos-version-min=$(OSX_MIN_VERSION) +darwin_CXXFLAGS=-pipe -std=$(CXX_STANDARD) -mmacos-version-min=$(OSX_MIN_VERSION) darwin_LDFLAGS=-Wl,-platform_version,macos,$(OSX_MIN_VERSION),$(OSX_SDK_VERSION) ifneq ($(build_os),darwin) -darwin_CFLAGS += -mlinker-version=$(LD64_VERSION) -darwin_CXXFLAGS += -mlinker-version=$(LD64_VERSION) +darwin_CFLAGS += -mlinker-version=$(LLD_VERSION) +darwin_CXXFLAGS += -mlinker-version=$(LLD_VERSION) darwin_LDFLAGS += -Wl,-no_adhoc_codesign -fuse-ld=lld endif diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index 7f0389b30d..938e9971ba 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -1,6 +1,6 @@ package=boost $(package)_version=1.81.0 -$(package)_download_path=https://boostorg.jfrog.io/artifactory/main/release/$($(package)_version)/source/ +$(package)_download_path=https://archives.boost.io/release/$($(package)_version)/source/ $(package)_file_name=boost_$(subst .,_,$($(package)_version)).tar.gz $(package)_sha256_hash=205666dea9f6a7cfed87c7a6dfbeb52a2c1b9de55712c9c1a87735d7181452b6 diff --git a/depends/packages/native_llvm.mk b/depends/packages/native_llvm.mk deleted file mode 100644 index c701147edc..0000000000 --- a/depends/packages/native_llvm.mk +++ /dev/null @@ -1,31 +0,0 @@ -package=native_llvm -$(package)_version=17.0.6 -$(package)_major_version=$(firstword $(subst ., ,$($(package)_version))) -$(package)_download_path=https://github.com/llvm/llvm-project/releases/download/llvmorg-$($(package)_version) -ifneq (,$(findstring aarch64,$(BUILD))) -$(package)_file_name=clang+llvm-$($(package)_version)-aarch64-linux-gnu.tar.xz -$(package)_sha256_hash=6dd62762285326f223f40b8e4f2864b5c372de3f7de0731cb7cd55ca5287b75a -else -$(package)_file_name=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-22.04.tar.xz -$(package)_sha256_hash=884ee67d647d77e58740c1e645649e29ae9e8a6fe87c1376be0f3a30f3cc9ab3 -endif - -define $(package)_stage_cmds - mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_major_version)/include && \ - mkdir -p $($(package)_staging_prefix_dir)/bin && \ - mkdir -p $($(package)_staging_prefix_dir)/include/llvm-c && \ - cp bin/clang $($(package)_staging_prefix_dir)/bin/ && \ - cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ && \ - cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/dsymutil && \ - cp bin/ld64.lld $($(package)_staging_prefix_dir)/bin/ld64.lld && \ - cp bin/llvm-ar $($(package)_staging_prefix_dir)/bin/llvm-ar && \ - cp bin/llvm-config $($(package)_staging_prefix_dir)/bin/ && \ - cp bin/llvm-nm $($(package)_staging_prefix_dir)/bin/llvm-nm && \ - cp bin/llvm-objdump $($(package)_staging_prefix_dir)/bin/llvm-objdump && \ - cp bin/llvm-ranlib $($(package)_staging_prefix_dir)/bin/llvm-ranlib && \ - cp bin/llvm-strip $($(package)_staging_prefix_dir)/bin/llvm-strip && \ - cp include/llvm-c/ExternC.h $($(package)_staging_prefix_dir)/include/llvm-c && \ - cp include/llvm-c/lto.h $($(package)_staging_prefix_dir)/include/llvm-c && \ - cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ - cp -r lib/clang/$($(package)_major_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_major_version)/include/ -endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index ca54093339..01ed0d7a92 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -24,13 +24,3 @@ multiprocess_packages = libmultiprocess capnp multiprocess_native_packages = native_libmultiprocess native_capnp usdt_linux_packages=systemtap - -darwin_native_packages = - -ifneq ($(build_os),darwin) - -ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) -darwin_native_packages+= native_llvm -endif - -endif diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 146e895c5a..d35139dd2d 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -1,9 +1,9 @@ package=qt -$(package)_version=5.15.13 +$(package)_version=5.15.14 $(package)_download_path=https://download.qt.io/official_releases/qt/5.15/$($(package)_version)/submodules $(package)_suffix=everywhere-opensource-src-$($(package)_version).tar.xz $(package)_file_name=qtbase-$($(package)_suffix) -$(package)_sha256_hash=4cca51dcc1f22ceeee6b3e33cd1c3a60b14e85e24644dca3af89a2c2989ab809 +$(package)_sha256_hash=500d3b390048e9538c28b5f523dfea6936f9c2e10d24ab46580ff57d430b98be $(package)_linux_dependencies=freetype fontconfig libxcb libxkbcommon libxcb_util libxcb_util_render libxcb_util_keysyms libxcb_util_image libxcb_util_wm $(package)_qt_libs=corelib network widgets gui plugins testlib $(package)_linguist_tools = lrelease lupdate lconvert @@ -20,15 +20,16 @@ $(package)_patches += duplicate_lcqpafonts.patch $(package)_patches += guix_cross_lib_path.patch $(package)_patches += fix-macos-linker.patch $(package)_patches += memory_resource.patch +$(package)_patches += clang_18_libpng.patch $(package)_patches += utc_from_string_no_optimize.patch $(package)_patches += windows_lto.patch $(package)_patches += zlib-timebits64.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) -$(package)_qttranslations_sha256_hash=24d4c58bc2a40c0f44f59ee64af4192c7d0038c1e45af61646cfc5b65058f271 +$(package)_qttranslations_sha256_hash=5b94d1a11b566908622fcca2f8b799744d2f8a68da20be4caa5953ed63b10489 $(package)_qttools_file_name=qttools-$($(package)_suffix) -$(package)_qttools_sha256_hash=57c9794c572c4e02871f2e7581525752b0cf85ea16cfab23a4ac9ba7b39a5d34 +$(package)_qttools_sha256_hash=12061a85baf5f4de8fbc795e1d3872b706f340211b9e70962caeffc6f5e89563 $(package)_extra_sources = $($(package)_qttranslations_file_name) $(package)_extra_sources += $($(package)_qttools_file_name) @@ -229,6 +230,7 @@ define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ patch -p1 -i $($(package)_patch_dir)/memory_resource.patch && \ patch -p1 -i $($(package)_patch_dir)/no_warnings_for_symbols.patch && \ + patch -p1 -i $($(package)_patch_dir)/clang_18_libpng.patch && \ patch -p1 -i $($(package)_patch_dir)/rcc_hardcode_timestamp.patch && \ patch -p1 -i $($(package)_patch_dir)/duplicate_lcqpafonts.patch && \ patch -p1 -i $($(package)_patch_dir)/utc_from_string_no_optimize.patch && \ diff --git a/depends/patches/qt/clang_18_libpng.patch b/depends/patches/qt/clang_18_libpng.patch new file mode 100644 index 0000000000..e807905b32 --- /dev/null +++ b/depends/patches/qt/clang_18_libpng.patch @@ -0,0 +1,40 @@ +fix Qt macOS build with Clang 18 + + See: + https://github.com/pnggroup/libpng/commit/893b8113f04d408cc6177c6de19c9889a48faa24. + + In a similar manner as zlib (madler/zlib#895), + libpng contains a header configuration that's no longer valid and + hasn't been exercised for the macOS target. + + - The target OS conditional macros are misused. Specifically + `TARGET_OS_MAC` covers all Apple targets, including iOS, and it + should not be checked with `#if defined` as they would always be + defined (to either 1 or 0) on Apple platforms. + - `#include <fp.h>` no longer works for the macOS target and results + in a compilation failure. macOS ships all required functions in + `math.h`, and clients should use `math.h` instead. + +--- a/qtbase/src/3rdparty/libpng/pngpriv.h ++++ b/qtbase/src/3rdparty/libpng/pngpriv.h +@@ -514,18 +514,8 @@ + */ + # include <float.h> + +-# if (defined(__MWERKS__) && defined(macintosh)) || defined(applec) || \ +- defined(THINK_C) || defined(__SC__) || defined(TARGET_OS_MAC) +- /* We need to check that <math.h> hasn't already been included earlier +- * as it seems it doesn't agree with <fp.h>, yet we should really use +- * <fp.h> if possible. +- */ +-# if !defined(__MATH_H__) && !defined(__MATH_H) && !defined(__cmath__) +-# include <fp.h> +-# endif +-# else +-# include <math.h> +-# endif ++# include <math.h> ++ + # if defined(_AMIGA) && defined(__SASC) && defined(_M68881) + /* Amiga SAS/C: We must include builtin FPU functions when compiling using + * MATH=68881 diff --git a/doc/JSON-RPC-interface.md b/doc/JSON-RPC-interface.md index 7640102172..10d8ee52eb 100644 --- a/doc/JSON-RPC-interface.md +++ b/doc/JSON-RPC-interface.md @@ -33,10 +33,10 @@ requests when multiple wallets are in use. ```sh # Get block count from the / endpoint when rpcuser=alice and rpcport=38332 -$ curl --user alice --data-binary '{"jsonrpc": "1.0", "id": "0", "method": "getblockcount", "params": []}' -H 'content-type: text/plain;' localhost:38332/ +$ curl --user alice --data-binary '{"jsonrpc": "2.0", "id": "0", "method": "getblockcount", "params": []}' -H 'content-type: application/json' localhost:38332/ # Get balance from the /wallet/walletname endpoint when rpcuser=alice, rpcport=38332 and rpcwallet=desc-wallet -$ curl --user alice --data-binary '{"jsonrpc": "1.0", "id": "0", "method": "getbalance", "params": []}' -H 'content-type: text/plain;' localhost:38332/wallet/desc-wallet +$ curl --user alice --data-binary '{"jsonrpc": "2.0", "id": "0", "method": "getbalance", "params": []}' -H 'content-type: application/json' localhost:38332/wallet/desc-wallet ``` @@ -80,7 +80,7 @@ The server recognizes [JSON-RPC v2.0](https://www.jsonrpc.org/specification) req and responds accordingly. A 2.0 request is identified by the presence of `"jsonrpc": "2.0"` in the request body. If that key + value is not present in a request, the legacy JSON-RPC v1.1 protocol is followed instead, which was the only available -protocol in previous releases. +protocol in v27.0 and prior releases. || 1.1 | 2.0 | |-|-|-| @@ -88,7 +88,7 @@ protocol in previous releases. | Response marker | (none) | `"jsonrpc": "2.0"` | | `"error"` and `"result"` fields in response | both present | only one is present | | HTTP codes in response | `200` unless there is any kind of RPC error (invalid parameters, method not found, etc) | Always `200` unless there is an actual HTTP server error (request parsing error, endpoint not found, etc) | -| Notifications: requests that get no reply | (not supported) | Supported for requests that exclude the "id" field | +| Notifications: requests that get no reply | (not supported) | Supported for requests that exclude the "id" field. Returns HTTP status `204` "No Content" | ## Security diff --git a/doc/bips.md b/doc/bips.md index 8309ee7e92..19a8091f55 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -69,3 +69,4 @@ BIPs that are implemented by Bitcoin Core: [`385`](https://github.com/bitcoin/bips/blob/master/bip-0385.mediawiki): Output Script Descriptors, and most of Script Expressions are implemented as of **v0.17.0** ([PR 13697](https://github.com/bitcoin/bitcoin/pull/13697)). * [`BIP 386`](https://github.com/bitcoin/bips/blob/master/bip-0386.mediawiki): tr() Output Script Descriptors are implemented as of **v22.0** ([PR 22051](https://github.com/bitcoin/bitcoin/pull/22051)). +* [`BIP 431`](https://github.com/bitcoin/bips/blob/master/bip-0431.mediawiki): transactions with nVersion=3 are standard and treated as Topologically Restricted Until Confirmation as of **v28.0** ([PR 29496](https://github.com/bitcoin/bitcoin/pull/29496)). diff --git a/doc/dependencies.md b/doc/dependencies.md index 2b9d9128d3..fc574b6164 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -30,7 +30,7 @@ You can find installation instructions in the `build-*.md` file for your platfor | [Fontconfig](../depends/packages/fontconfig.mk) | [link](https://www.freedesktop.org/wiki/Software/fontconfig/) | [2.12.6](https://github.com/bitcoin/bitcoin/pull/23495) | 2.6 | Yes | | [FreeType](../depends/packages/freetype.mk) | [link](https://freetype.org) | [2.11.0](https://github.com/bitcoin/bitcoin/commit/01544dd78ccc0b0474571da854e27adef97137fb) | 2.3.0 | Yes | | [qrencode](../depends/packages/qrencode.mk) | [link](https://fukuchi.org/works/qrencode/) | [4.1.1](https://github.com/bitcoin/bitcoin/pull/27312) | | No | -| [Qt](../depends/packages/qt.mk) | [link](https://download.qt.io/official_releases/qt/) | [5.15.13](https://github.com/bitcoin/bitcoin/pull/29732) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | +| [Qt](../depends/packages/qt.mk) | [link](https://download.qt.io/official_releases/qt/) | [5.15.14](https://github.com/bitcoin/bitcoin/pull/30198) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | ### Networking | Dependency | Releases | Version used | Minimum required | Runtime | diff --git a/doc/design/libraries.md b/doc/design/libraries.md index 251c52199d..aa8034ab37 100644 --- a/doc/design/libraries.md +++ b/doc/design/libraries.md @@ -5,6 +5,7 @@ | *libbitcoin_cli* | RPC client functionality used by *bitcoin-cli* executable | | *libbitcoin_common* | Home for common functionality shared by different executables and libraries. Similar to *libbitcoin_util*, but higher-level (see [Dependencies](#dependencies)). | | *libbitcoin_consensus* | Stable, backwards-compatible consensus functionality used by *libbitcoin_node* and *libbitcoin_wallet*. | +| *libbitcoin_crypto* | Hardware-optimized functions for data encryption, hashing, message authentication, and key derivation. | | *libbitcoin_kernel* | Consensus engine and support library used for validation by *libbitcoin_node*. | | *libbitcoinqt* | GUI functionality used by *bitcoin-qt* and *bitcoin-gui* executables. | | *libbitcoin_ipc* | IPC functionality used by *bitcoin-node*, *bitcoin-wallet*, *bitcoin-gui* executables to communicate when [`--enable-multiprocess`](multiprocess.md) is used. | @@ -53,13 +54,18 @@ bitcoin-wallet[bitcoin-wallet]-->libbitcoin_wallet_tool; libbitcoin_cli-->libbitcoin_util; libbitcoin_cli-->libbitcoin_common; +libbitcoin_consensus-->libbitcoin_crypto; + libbitcoin_common-->libbitcoin_consensus; +libbitcoin_common-->libbitcoin_crypto; libbitcoin_common-->libbitcoin_util; libbitcoin_kernel-->libbitcoin_consensus; +libbitcoin_kernel-->libbitcoin_crypto; libbitcoin_kernel-->libbitcoin_util; libbitcoin_node-->libbitcoin_consensus; +libbitcoin_node-->libbitcoin_crypto; libbitcoin_node-->libbitcoin_kernel; libbitcoin_node-->libbitcoin_common; libbitcoin_node-->libbitcoin_util; @@ -67,7 +73,10 @@ libbitcoin_node-->libbitcoin_util; libbitcoinqt-->libbitcoin_common; libbitcoinqt-->libbitcoin_util; +libbitcoin_util-->libbitcoin_crypto; + libbitcoin_wallet-->libbitcoin_common; +libbitcoin_wallet-->libbitcoin_crypto; libbitcoin_wallet-->libbitcoin_util; libbitcoin_wallet_tool-->libbitcoin_wallet; @@ -78,22 +87,23 @@ class bitcoin-qt,bitcoind,bitcoin-cli,bitcoin-wallet bold ``` </td></tr><tr><td> -**Dependency graph**. Arrows show linker symbol dependencies. *Consensus* lib depends on nothing. *Util* lib is depended on by everything. *Kernel* lib depends only on consensus and util. +**Dependency graph**. Arrows show linker symbol dependencies. *Crypto* lib depends on nothing. *Util* lib is depended on by everything. *Kernel* lib depends only on consensus, crypto, and util. </td></tr></table> - The graph shows what _linker symbols_ (functions and variables) from each library other libraries can call and reference directly, but it is not a call graph. For example, there is no arrow connecting *libbitcoin_wallet* and *libbitcoin_node* libraries, because these libraries are intended to be modular and not depend on each other's internal implementation details. But wallet code is still able to call node code indirectly through the `interfaces::Chain` abstract class in [`interfaces/chain.h`](../../src/interfaces/chain.h) and node code calls wallet code through the `interfaces::ChainClient` and `interfaces::Chain::Notifications` abstract classes in the same file. In general, defining abstract classes in [`src/interfaces/`](../../src/interfaces/) can be a convenient way of avoiding unwanted direct dependencies or circular dependencies between libraries. -- *libbitcoin_consensus* should be a standalone dependency that any library can depend on, and it should not depend on any other libraries itself. +- *libbitcoin_crypto* should be a standalone dependency that any library can depend on, and it should not depend on any other libraries itself. -- *libbitcoin_util* should also be a standalone dependency that any library can depend on, and it should not depend on other internal libraries. +- *libbitcoin_consensus* should only depend on *libbitcoin_crypto*, and all other libraries besides *libbitcoin_crypto* should be allowed to depend on it. -- *libbitcoin_common* should serve a similar function as *libbitcoin_util* and be a place for miscellaneous code used by various daemon, GUI, and CLI applications and libraries to live. It should not depend on anything other than *libbitcoin_util* and *libbitcoin_consensus*. The boundary between _util_ and _common_ is a little fuzzy but historically _util_ has been used for more generic, lower-level things like parsing hex, and _common_ has been used for bitcoin-specific, higher-level things like parsing base58. The difference between util and common is mostly important because *libbitcoin_kernel* is not supposed to depend on *libbitcoin_common*, only *libbitcoin_util*. In general, if it is ever unclear whether it is better to add code to *util* or *common*, it is probably better to add it to *common* unless it is very generically useful or useful particularly to include in the kernel. +- *libbitcoin_util* should be a standalone dependency that any library can depend on, and it should not depend on other libraries except *libbitcoin_crypto*. It provides basic utilities that fill in gaps in the C++ standard library and provide lightweight abstractions over platform-specific features. Since the util library is distributed with the kernel and is usable by kernel applications, it shouldn't contain functions that external code shouldn't call, like higher level code targetted at the node or wallet. (*libbitcoin_common* is a better place for higher level code, or code that is meant to be used by internal applications only.) +- *libbitcoin_common* is a home for miscellaneous shared code used by different Bitcoin Core applications. It should not depend on anything other than *libbitcoin_util*, *libbitcoin_consensus*, and *libbitcoin_crypto*. -- *libbitcoin_kernel* should only depend on *libbitcoin_util* and *libbitcoin_consensus*. +- *libbitcoin_kernel* should only depend on *libbitcoin_util*, *libbitcoin_consensus*, and *libbitcoin_crypto*. -- The only thing that should depend on *libbitcoin_kernel* internally should be *libbitcoin_node*. GUI and wallet libraries *libbitcoinqt* and *libbitcoin_wallet* in particular should not depend on *libbitcoin_kernel* and the unneeded functionality it would pull in, like block validation. To the extent that GUI and wallet code need scripting and signing functionality, they should be get able it from *libbitcoin_consensus*, *libbitcoin_common*, and *libbitcoin_util*, instead of *libbitcoin_kernel*. +- The only thing that should depend on *libbitcoin_kernel* internally should be *libbitcoin_node*. GUI and wallet libraries *libbitcoinqt* and *libbitcoin_wallet* in particular should not depend on *libbitcoin_kernel* and the unneeded functionality it would pull in, like block validation. To the extent that GUI and wallet code need scripting and signing functionality, they should be get able it from *libbitcoin_consensus*, *libbitcoin_common*, *libbitcoin_crypto*, and *libbitcoin_util*, instead of *libbitcoin_kernel*. - GUI, node, and wallet code internal implementations should all be independent of each other, and the *libbitcoinqt*, *libbitcoin_node*, *libbitcoin_wallet* libraries should never reference each other's symbols. They should only call each other through [`src/interfaces/`](../../src/interfaces/) abstract interfaces. diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 3458a71467..eb2bb41aa4 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -13,7 +13,6 @@ Developer Notes - [Development tips and tricks](#development-tips-and-tricks) - [Compiling for debugging](#compiling-for-debugging) - [Show sources in debugging](#show-sources-in-debugging) - - [Compiling for gprof profiling](#compiling-for-gprof-profiling) - [`debug.log`](#debuglog) - [Signet, testnet, and regtest modes](#signet-testnet-and-regtest-modes) - [DEBUG_LOCKORDER](#debug_lockorder) @@ -386,10 +385,6 @@ ln -s /path/to/project/root/src src 3. Use `debugedit` to modify debug information in the binary. -### Compiling for gprof profiling - -Run configure with the `--enable-gprof` option, then make. - ### `debug.log` If the code is behaving strangely, take a look in the `debug.log` file in the data directory; @@ -497,6 +492,10 @@ make cov # unit and functional tests. ``` +Additional LCOV options can be specified using `LCOV_OPTS`, but may be dependant +on the version of LCOV. For example, when using LCOV `2.x`, branch coverage can be +enabled by setting `LCOV_OPTS="--rc branch_coverage=1"`, when configuring. + ### Performance profiling with perf Profiling is a good way to get a precise idea of where time is being spent in diff --git a/doc/init.md b/doc/init.md index 7f79027718..64ab971557 100644 --- a/doc/init.md +++ b/doc/init.md @@ -43,8 +43,8 @@ This allows for running bitcoind without having to do any manual configuration. `conf`, `pid`, and `wallet` accept relative paths which are interpreted as relative to the data directory. `wallet` *only* supports relative paths. -For an example configuration file that describes the configuration settings, -see `share/examples/bitcoin.conf`. +To generate an example configuration file that describes the configuration settings, +see [contrib/devtools/README.md](../contrib/devtools/README.md#gen-bitcoin-confsh). Paths --------------------------------- diff --git a/doc/policy/mempool-replacements.md b/doc/policy/mempool-replacements.md index 3fd7ff2ad2..d5642eaccc 100644 --- a/doc/policy/mempool-replacements.md +++ b/doc/policy/mempool-replacements.md @@ -12,7 +12,7 @@ other consensus and policy rules, each of the following conditions are met: 1. The directly conflicting transactions all signal replaceability explicitly. A transaction is signaling BIP125 replaceability if any of its inputs have an nSequence number less than (0xffffffff - 1). - A transaction also signals replaceability if its nVersion field is set to 3. + A transaction also signals replaceability if its version field is set to 3. *Rationale*: See [BIP125 explanation](https://github.com/bitcoin/bips/blob/master/bip-0125.mediawiki#motivation). diff --git a/doc/policy/packages.md b/doc/policy/packages.md index 7e983221c5..a220bdd17f 100644 --- a/doc/policy/packages.md +++ b/doc/policy/packages.md @@ -36,10 +36,29 @@ The following rules are enforced for all packages: * Packages cannot have conflicting transactions, i.e. no two transactions in a package can spend the same inputs. Packages cannot have duplicate transactions. (#20833) -* No transaction in a package can conflict with a mempool transaction. Replace By Fee is - currently disabled for packages. (#20833) +* Only limited package replacements are currently considered. (#28984) - - Package RBF may be enabled in the future. + - All direct conflicts must signal replacement (or have `-mempoolfullrbf=1` set). + + - Packages are 1-parent-1-child, with no in-mempool ancestors of the package. + + - All conflicting clusters(connected components of mempool transactions) must be clusters of up to size 2. + + - No more than MAX_REPLACEMENT_CANDIDATES transactions can be replaced, analogous to + regular [replacement rule](./mempool-replacements.md) 5). + + - Replacements must pay more total total fees at the incremental relay fee (analogous to + regular [replacement rules](./mempool-replacements.md) 3 and 4). + + - Parent feerate must be lower than package feerate. + + - Must improve [feerate diagram](https://delvingbitcoin.org/t/mempool-incentive-compatibility/553). (#29242) + + - *Rationale*: Basic support for package RBF can be used by wallets + by making chains of no longer than two, then directly conflicting + those chains when needed. Combined with V3 transactions this can + result in more robust fee bumping. More general package RBF may be + enabled in the future. * When packages are evaluated against ancestor/descendant limits, the union of all transactions' descendants and ancestors is considered. (#21800) diff --git a/doc/release-28984.md b/doc/release-28984.md new file mode 100644 index 0000000000..3da64f6578 --- /dev/null +++ b/doc/release-28984.md @@ -0,0 +1,6 @@ +P2P and network changes +----------------------- + +- Limited package RBF is now enabled, where the proposed conflicting package would result in + a connected component, aka cluster, of size 2 in the mempool. All clusters being conflicted + against must be of size 2 or lower. diff --git a/doc/release-notes-27101.md b/doc/release-notes-27101.md index 8775b59c00..7ce1e9a8c1 100644 --- a/doc/release-notes-27101.md +++ b/doc/release-notes-27101.md @@ -2,8 +2,5 @@ JSON-RPC -------- The JSON-RPC server now recognizes JSON-RPC 2.0 requests and responds with -strict adherence to the specification (https://www.jsonrpc.org/specification): - -- Returning HTTP "204 No Content" responses to JSON-RPC 2.0 notifications instead of full responses. -- Returning HTTP "200 OK" responses in all other cases, rather than 404 responses for unknown methods, 500 responses for invalid parameters, etc. -- Returning either "result" fields or "error" fields in JSON-RPC responses, rather than returning both fields with one field set to null. +strict adherence to the [specification](https://www.jsonrpc.org/specification). +See [JSON-RPC-interface.md](/doc/JSON-RPC-interface.md#json-rpc-11-vs-20) for details.
\ No newline at end of file diff --git a/doc/release-notes-29091-29165.md b/doc/release-notes-29091-29165.md new file mode 100644 index 0000000000..9c9f8e4e50 --- /dev/null +++ b/doc/release-notes-29091-29165.md @@ -0,0 +1,5 @@ +Build +----- + +GCC 11.1 or later, or Clang 15+ or later, +are now required to compile Bitcoin Core. diff --git a/doc/release-notes-29612.md b/doc/release-notes-29612.md new file mode 100644 index 0000000000..31af3cab09 --- /dev/null +++ b/doc/release-notes-29612.md @@ -0,0 +1,8 @@ +RPC +--- + +- The `dumptxoutset` RPC now returns the UTXO set dump in a new and + improved format. At the same time the `loadtxoutset` RPC now + expects this new format in dumps it tries to load. Dumps with the + old format are no longer supported and need to be recreated using + the new format in order to be usable. diff --git a/doc/release-notes-30058.md b/doc/release-notes-30058.md new file mode 100644 index 0000000000..47e7ae704e --- /dev/null +++ b/doc/release-notes-30058.md @@ -0,0 +1,7 @@ +- When running with -alertnotify, an alert can now be raised multiple +times instead of just once. Previously, it was only raised when unknown +new consensus rules were activated, whereas the scope has now been +increased to include all kernel warnings. Specifically, alerts will now +also be raised when an invalid chain with a large amount of work has +been detected. Additional warnings may be added in the future. +(#30058) diff --git a/doc/release-notes-30192.md b/doc/release-notes-30192.md new file mode 100644 index 0000000000..2a6c17d455 --- /dev/null +++ b/doc/release-notes-30192.md @@ -0,0 +1,6 @@ +Build +----- + +`--enable-lcov-branch-coverage` has been removed, given +incompatibilities between lcov version 1 & 2. `LCOV_OPTS` +should be used to set any options instead. diff --git a/doc/release-notes/release-notes-27.1.md b/doc/release-notes/release-notes-27.1.md new file mode 100644 index 0000000000..b19d70da33 --- /dev/null +++ b/doc/release-notes/release-notes-27.1.md @@ -0,0 +1,114 @@ +27.1 Release Notes +===================== + +Bitcoin Core version 27.1 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-27.1/> + +This release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux Kernel 3.17+, macOS 11.0+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +### Miniscript + +- #29853 sign: don't assume we are parsing a sane TapMiniscript + +### RPC + +- #29869 rpc, bugfix: Enforce maximum value for setmocktime +- #29870 rpc: Reword SighashFromStr error message +- #30094 rpc: move UniValue in blockToJSON + +### Index + +- #29776 Fix #29767, set m_synced = true after Commit() + +### Gui + +- #gui812 Fix create unsigned transaction fee bump +- #gui813 Don't permit port in proxy IP option + +### Test + +- #29892 test: Fix failing univalue float test + +### P2P + +- #30085 p2p: detect addnode cjdns peers in GetAddedNodeInfo() + +### Build + +- #29747 depends: fix mingw-w64 Qt DEBUG=1 build +- #29859 build: Fix false positive CHECK_ATOMIC test +- #29985 depends: Fix build of Qt for 32-bit platforms with recent glibc +- #30097 crypto: disable asan for sha256_sse4 with clang and -O0 +- #30151 depends: Fetch miniupnpc sources from an alternative website +- #30216 build: Fix building fuzz binary on on SunOS / illumos +- #30217 depends: Update Boost download link + +### Doc + +- #29934 doc: add LLVM instruction for macOS < 13 + +### CI + +- #29856 ci: Bump s390x to ubuntu:24.04 + +### Misc + +- #29691 Change Luke Dashjr seed to dashjr-list-of-p2p-nodes.us +- #30149 contrib: Renew Windows code signing certificate + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Antoine Poinsot +- Ava Chow +- Cory Fields +- dergoegge +- fanquake +- furszy +- Hennadii Stepanov +- Jon Atack +- laanwj +- Luke Dashjr +- MarcoFalke +- nanlour +- Sjors Provoost +- willcl-ark + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/src/.clang-tidy b/src/.clang-tidy index a00400f083..61adce1d50 100644 --- a/src/.clang-tidy +++ b/src/.clang-tidy @@ -2,6 +2,7 @@ Checks: ' -*, bitcoin-*, bugprone-argument-comment, +bugprone-move-forwarding-reference, bugprone-string-constructor, bugprone-use-after-move, bugprone-lambda-function-name, diff --git a/src/Makefile.am b/src/Makefile.am index ad37928b4d..4a1973aa87 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -8,8 +8,8 @@ print-%: FORCE DIST_SUBDIRS = secp256k1 -AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) $(CORE_LDFLAGS) -AM_CXXFLAGS = $(CORE_CXXFLAGS) $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) +AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(SANITIZER_LDFLAGS) $(CORE_LDFLAGS) +AM_CXXFLAGS = $(CORE_CXXFLAGS) $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(SANITIZER_CXXFLAGS) AM_OBJCXXFLAGS = $(AM_CXXFLAGS) AM_CPPFLAGS = $(DEBUG_CPPFLAGS) $(HARDENED_CPPFLAGS) $(CORE_CPPFLAGS) AM_LIBTOOLFLAGS = --preserve-dup-deps @@ -137,13 +137,16 @@ BITCOIN_CORE_H = \ common/bloom.h \ common/init.h \ common/run_command.h \ + common/types.h \ common/url.h \ compat/assumptions.h \ compat/byteswap.h \ compat/compat.h \ compat/cpuid.h \ compat/endian.h \ + common/messages.h \ common/settings.h \ + common/signmessage.h \ common/system.h \ compressor.h \ consensus/consensus.h \ @@ -193,6 +196,7 @@ BITCOIN_CORE_H = \ kernel/messagestartchars.h \ kernel/notifications_interface.h \ kernel/validation_cache_sizes.h \ + kernel/warning.h \ key.h \ key_io.h \ logging.h \ @@ -233,8 +237,10 @@ BITCOIN_CORE_H = \ node/timeoffsets.h \ node/transaction.h \ node/txreconciliation.h \ + node/types.h \ node/utxo_snapshot.h \ node/validation_cache_args.h \ + node/warnings.h \ noui.h \ outputtype.h \ policy/v3_policy.h \ @@ -267,6 +273,7 @@ BITCOIN_CORE_H = \ script/descriptor.h \ script/keyorigin.h \ script/miniscript.h \ + script/parsing.h \ script/sigcache.h \ script/sign.h \ script/signingprovider.h \ @@ -292,15 +299,14 @@ BITCOIN_CORE_H = \ util/batchpriority.h \ util/bip32.h \ util/bitdeque.h \ + util/bitset.h \ util/bytevectorhash.h \ util/chaintype.h \ util/check.h \ util/epochguard.h \ - util/error.h \ util/exception.h \ util/fastrange.h \ util/feefrac.h \ - util/fees.h \ util/fs.h \ util/fs_helpers.h \ util/golombrice.h \ @@ -308,7 +314,6 @@ BITCOIN_CORE_H = \ util/hasher.h \ util/insert.h \ util/macros.h \ - util/message.h \ util/moneystr.h \ util/overflow.h \ util/overloaded.h \ @@ -318,7 +323,7 @@ BITCOIN_CORE_H = \ util/serfloat.h \ util/signalinterrupt.h \ util/sock.h \ - util/spanparsing.h \ + util/strencodings.h \ util/string.h \ util/subprocess.h \ util/syserror.h \ @@ -333,6 +338,7 @@ BITCOIN_CORE_H = \ util/translation.h \ util/types.h \ util/ui_change_type.h \ + util/vecdeque.h \ util/vector.h \ validation.h \ validationinterface.h \ @@ -363,7 +369,6 @@ BITCOIN_CORE_H = \ wallet/wallettool.h \ wallet/walletutil.h \ walletinitinterface.h \ - warnings.h \ zmq/zmqabstractnotifier.h \ zmq/zmqnotificationinterface.h \ zmq/zmqpublishnotifier.h \ @@ -440,6 +445,7 @@ libbitcoin_node_a_SOURCES = \ node/txreconciliation.cpp \ node/utxo_snapshot.cpp \ node/validation_cache_args.cpp \ + node/warnings.cpp \ noui.cpp \ policy/v3_policy.cpp \ policy/fees.cpp \ @@ -567,6 +573,8 @@ crypto_libbitcoin_crypto_base_la_SOURCES = \ crypto/chacha20poly1305.h \ crypto/chacha20poly1305.cpp \ crypto/common.h \ + crypto/hex_base.cpp \ + crypto/hex_base.h \ crypto/hkdf_sha256_32.cpp \ crypto/hkdf_sha256_32.h \ crypto/hmac_sha256.cpp \ @@ -589,7 +597,8 @@ crypto_libbitcoin_crypto_base_la_SOURCES = \ crypto/sha512.cpp \ crypto/sha512.h \ crypto/siphash.cpp \ - crypto/siphash.h + crypto/siphash.h \ + support/cleanse.cpp # See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and # CXXFLAGS above @@ -659,9 +668,7 @@ libbitcoin_consensus_a_SOURCES = \ span.h \ tinyformat.h \ uint256.cpp \ - uint256.h \ - util/strencodings.cpp \ - util/strencodings.h + uint256.h # # common # @@ -671,6 +678,7 @@ libbitcoin_common_a_SOURCES = \ addresstype.cpp \ base58.cpp \ bech32.cpp \ + chainparamsbase.cpp \ chainparams.cpp \ coins.cpp \ common/args.cpp \ @@ -678,8 +686,10 @@ libbitcoin_common_a_SOURCES = \ common/config.cpp \ common/init.cpp \ common/interfaces.cpp \ + common/messages.cpp \ common/run_command.cpp \ common/settings.cpp \ + common/signmessage.cpp \ common/system.cpp \ common/url.cpp \ compressor.cpp \ @@ -709,10 +719,10 @@ libbitcoin_common_a_SOURCES = \ scheduler.cpp \ script/descriptor.cpp \ script/miniscript.cpp \ + script/parsing.cpp \ script/sign.cpp \ script/signingprovider.cpp \ script/solver.cpp \ - warnings.cpp \ $(BITCOIN_CORE_H) # @@ -721,13 +731,11 @@ libbitcoin_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_util_a_SOURCES = \ support/lockedpool.cpp \ - chainparamsbase.cpp \ clientversion.cpp \ logging.cpp \ random.cpp \ randomenv.cpp \ streams.cpp \ - support/cleanse.cpp \ sync.cpp \ util/asmap.cpp \ util/batchpriority.cpp \ @@ -735,16 +743,13 @@ libbitcoin_util_a_SOURCES = \ util/bytevectorhash.cpp \ util/chaintype.cpp \ util/check.cpp \ - util/error.cpp \ util/exception.cpp \ util/feefrac.cpp \ - util/fees.cpp \ util/fs.cpp \ util/fs_helpers.cpp \ util/hasher.cpp \ util/sock.cpp \ util/syserror.cpp \ - util/message.cpp \ util/moneystr.cpp \ util/rbf.cpp \ util/readwritefile.cpp \ @@ -753,7 +758,6 @@ libbitcoin_util_a_SOURCES = \ util/threadinterrupt.cpp \ util/threadnames.cpp \ util/serfloat.cpp \ - util/spanparsing.cpp \ util/strencodings.cpp \ util/string.cpp \ util/time.cpp \ @@ -970,7 +974,6 @@ libbitcoinkernel_la_SOURCES = \ script/solver.cpp \ signet.cpp \ streams.cpp \ - support/cleanse.cpp \ support/lockedpool.cpp \ sync.cpp \ txdb.cpp \ @@ -994,8 +997,7 @@ libbitcoinkernel_la_SOURCES = \ util/tokenpipe.cpp \ validation.cpp \ validationinterface.cpp \ - versionbits.cpp \ - warnings.cpp + versionbits.cpp # Required for obj/build.h to be generated first. # More details: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 7ba0111fa6..2ba72c9e76 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -23,6 +23,7 @@ bench_bench_bitcoin_SOURCES = \ bench/ccoins_caching.cpp \ bench/chacha20.cpp \ bench/checkblock.cpp \ + bench/checkblockindex.cpp \ bench/checkqueue.cpp \ bench/crypto_hash.cpp \ bench/data.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 62a82189c1..633d0776f5 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -118,6 +118,7 @@ BITCOIN_TESTS =\ test/net_peer_eviction_tests.cpp \ test/net_tests.cpp \ test/netbase_tests.cpp \ + test/node_warnings_tests.cpp \ test/orphanage_tests.cpp \ test/peerman_tests.cpp \ test/pmt_tests.cpp \ @@ -203,6 +204,7 @@ endif FUZZ_WALLET_SRC = \ wallet/test/fuzz/coincontrol.cpp \ wallet/test/fuzz/coinselection.cpp \ + wallet/test/fuzz/crypter.cpp \ wallet/test/fuzz/fees.cpp \ wallet/test/fuzz/parse_iso8601.cpp \ wallet/test/fuzz/wallet_bdb_parser.cpp @@ -277,7 +279,7 @@ FUZZ_SUITE_LD_COMMON += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif if ENABLE_FUZZ_BINARY -test_fuzz_fuzz_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) +test_fuzz_fuzz_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) $(EVENT_CFLAGS) test_fuzz_fuzz_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_fuzz_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_fuzz_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) @@ -293,6 +295,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/bech32.cpp \ test/fuzz/bip324.cpp \ test/fuzz/bitdeque.cpp \ + test/fuzz/bitset.cpp \ test/fuzz/block.cpp \ test/fuzz/block_header.cpp \ test/fuzz/blockfilter.cpp \ @@ -326,6 +329,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/headerssync.cpp \ test/fuzz/hex.cpp \ test/fuzz/http_request.cpp \ + test/fuzz/i2p.cpp \ test/fuzz/integer.cpp \ test/fuzz/key.cpp \ test/fuzz/key_io.cpp \ @@ -372,6 +376,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/script_format.cpp \ test/fuzz/script_interpreter.cpp \ test/fuzz/script_ops.cpp \ + test/fuzz/script_parsing.cpp \ test/fuzz/script_sigcache.cpp \ test/fuzz/script_sign.cpp \ test/fuzz/scriptnum_ops.cpp \ @@ -381,7 +386,6 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/signet.cpp \ test/fuzz/socks5.cpp \ test/fuzz/span.cpp \ - test/fuzz/spanparsing.cpp \ test/fuzz/string.cpp \ test/fuzz/strprintf.cpp \ test/fuzz/system.cpp \ @@ -396,6 +400,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/utxo_snapshot.cpp \ test/fuzz/utxo_total_supply.cpp \ test/fuzz/validation_load_mempool.cpp \ + test/fuzz/vecdeque.cpp \ test/fuzz/versionbits.cpp endif # ENABLE_FUZZ_BINARY diff --git a/src/base58.cpp b/src/base58.cpp index cf5d62f164..f9165ed55f 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -14,6 +14,8 @@ #include <limits> +using util::ContainsNoNUL; + /** All alphanumeric characters except for "0", "I", "O", and "l" */ static const char* pszBase58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; static const int8_t mapBase58[256] = { diff --git a/src/bech32.cpp b/src/bech32.cpp index ba3c419d8b..d8d31a415c 100644 --- a/src/bech32.cpp +++ b/src/bech32.cpp @@ -19,6 +19,9 @@ namespace typedef std::vector<uint8_t> data; +/** The Bech32 and Bech32m checksum size */ +constexpr size_t CHECKSUM_SIZE = 6; + /** The Bech32 and Bech32m character set for encoding. */ const char* CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l"; @@ -308,18 +311,18 @@ bool CheckCharacters(const std::string& str, std::vector<int>& errors) return errors.empty(); } -/** Expand a HRP for use in checksum computation. */ -data ExpandHRP(const std::string& hrp) +std::vector<unsigned char> PreparePolynomialCoefficients(const std::string& hrp, const data& values) { data ret; - ret.reserve(hrp.size() + 90); - ret.resize(hrp.size() * 2 + 1); - for (size_t i = 0; i < hrp.size(); ++i) { - unsigned char c = hrp[i]; - ret[i] = c >> 5; - ret[i + hrp.size() + 1] = c & 0x1f; - } - ret[hrp.size()] = 0; + ret.reserve(hrp.size() + 1 + hrp.size() + values.size() + CHECKSUM_SIZE); + + /** Expand a HRP for use in checksum computation. */ + for (size_t i = 0; i < hrp.size(); ++i) ret.push_back(hrp[i] >> 5); + ret.push_back(0); + for (size_t i = 0; i < hrp.size(); ++i) ret.push_back(hrp[i] & 0x1f); + + ret.insert(ret.end(), values.begin(), values.end()); + return ret; } @@ -331,7 +334,8 @@ Encoding VerifyChecksum(const std::string& hrp, const data& values) // list of values would result in a new valid list. For that reason, Bech32 requires the // resulting checksum to be 1 instead. In Bech32m, this constant was amended. See // https://gist.github.com/sipa/14c248c288c3880a3b191f978a34508e for details. - const uint32_t check = PolyMod(Cat(ExpandHRP(hrp), values)); + auto enc = PreparePolynomialCoefficients(hrp, values); + const uint32_t check = PolyMod(enc); if (check == EncodingConstant(Encoding::BECH32)) return Encoding::BECH32; if (check == EncodingConstant(Encoding::BECH32M)) return Encoding::BECH32M; return Encoding::INVALID; @@ -340,11 +344,11 @@ Encoding VerifyChecksum(const std::string& hrp, const data& values) /** Create a checksum. */ data CreateChecksum(Encoding encoding, const std::string& hrp, const data& values) { - data enc = Cat(ExpandHRP(hrp), values); - enc.resize(enc.size() + 6); // Append 6 zeroes + auto enc = PreparePolynomialCoefficients(hrp, values); + enc.insert(enc.end(), CHECKSUM_SIZE, 0x00); uint32_t mod = PolyMod(enc) ^ EncodingConstant(encoding); // Determine what to XOR into those 6 zeroes. - data ret(6); - for (size_t i = 0; i < 6; ++i) { + data ret(CHECKSUM_SIZE); + for (size_t i = 0; i < CHECKSUM_SIZE; ++i) { // Convert the 5-bit groups in mod to checksum values. ret[i] = (mod >> (5 * (5 - i))) & 31; } @@ -359,22 +363,23 @@ std::string Encode(Encoding encoding, const std::string& hrp, const data& values // to return a lowercase Bech32/Bech32m string, but if given an uppercase HRP, the // result will always be invalid. for (const char& c : hrp) assert(c < 'A' || c > 'Z'); - data checksum = CreateChecksum(encoding, hrp, values); - data combined = Cat(values, checksum); - std::string ret = hrp + '1'; - ret.reserve(ret.size() + combined.size()); - for (const auto c : combined) { - ret += CHARSET[c]; - } + + std::string ret; + ret.reserve(hrp.size() + 1 + values.size() + CHECKSUM_SIZE); + ret += hrp; + ret += '1'; + for (const uint8_t& i : values) ret += CHARSET[i]; + for (const uint8_t& i : CreateChecksum(encoding, hrp, values)) ret += CHARSET[i]; return ret; } /** Decode a Bech32 or Bech32m string. */ -DecodeResult Decode(const std::string& str) { +DecodeResult Decode(const std::string& str, CharLimit limit) { std::vector<int> errors; if (!CheckCharacters(str, errors)) return {}; size_t pos = str.rfind('1'); - if (str.size() > 90 || pos == str.npos || pos == 0 || pos + 7 > str.size()) { + if (str.size() > limit) return {}; + if (pos == str.npos || pos == 0 || pos + CHECKSUM_SIZE >= str.size()) { return {}; } data values(str.size() - 1 - pos); @@ -388,21 +393,22 @@ DecodeResult Decode(const std::string& str) { values[i] = rev; } std::string hrp; + hrp.reserve(pos); for (size_t i = 0; i < pos; ++i) { hrp += LowerCase(str[i]); } Encoding result = VerifyChecksum(hrp, values); if (result == Encoding::INVALID) return {}; - return {result, std::move(hrp), data(values.begin(), values.end() - 6)}; + return {result, std::move(hrp), data(values.begin(), values.end() - CHECKSUM_SIZE)}; } /** Find index of an incorrect character in a Bech32 string. */ -std::pair<std::string, std::vector<int>> LocateErrors(const std::string& str) { +std::pair<std::string, std::vector<int>> LocateErrors(const std::string& str, CharLimit limit) { std::vector<int> error_locations{}; - if (str.size() > 90) { - error_locations.resize(str.size() - 90); - std::iota(error_locations.begin(), error_locations.end(), 90); + if (str.size() > limit) { + error_locations.resize(str.size() - limit); + std::iota(error_locations.begin(), error_locations.end(), static_cast<int>(limit)); return std::make_pair("Bech32 string too long", std::move(error_locations)); } @@ -414,12 +420,13 @@ std::pair<std::string, std::vector<int>> LocateErrors(const std::string& str) { if (pos == str.npos) { return std::make_pair("Missing separator", std::vector<int>{}); } - if (pos == 0 || pos + 7 > str.size()) { + if (pos == 0 || pos + CHECKSUM_SIZE >= str.size()) { error_locations.push_back(pos); return std::make_pair("Invalid separator position", std::move(error_locations)); } std::string hrp; + hrp.reserve(pos); for (size_t i = 0; i < pos; ++i) { hrp += LowerCase(str[i]); } @@ -441,9 +448,10 @@ std::pair<std::string, std::vector<int>> LocateErrors(const std::string& str) { std::optional<Encoding> error_encoding; for (Encoding encoding : {Encoding::BECH32, Encoding::BECH32M}) { std::vector<int> possible_errors; - // Recall that (ExpandHRP(hrp) ++ values) is interpreted as a list of coefficients of a polynomial + // Recall that (expanded hrp + values) is interpreted as a list of coefficients of a polynomial // over GF(32). PolyMod computes the "remainder" of this polynomial modulo the generator G(x). - uint32_t residue = PolyMod(Cat(ExpandHRP(hrp), values)) ^ EncodingConstant(encoding); + auto enc = PreparePolynomialCoefficients(hrp, values); + uint32_t residue = PolyMod(enc) ^ EncodingConstant(encoding); // All valid codewords should be multiples of G(x), so this remainder (after XORing with the encoding // constant) should be 0 - hence 0 indicates there are no errors present. diff --git a/src/bech32.h b/src/bech32.h index 5e89e6efda..fe2a276ae0 100644 --- a/src/bech32.h +++ b/src/bech32.h @@ -28,6 +28,14 @@ enum class Encoding { BECH32M, //!< Bech32m encoding as defined in BIP350 }; +/** Character limits for Bech32(m) encoded strings. Character limits are how we provide error location guarantees. + * These values should never exceed 2^31 - 1 (max value for a 32-bit int), since there are places where we may need to + * convert the CharLimit::VALUE to an int. In practice, this should never happen since this CharLimit applies to an address encoding + * and we would never encode an address with such a massive value */ +enum CharLimit : size_t { + BECH32 = 90, //!< BIP173/350 imposed character limit for Bech32(m) encoded addresses. This guarantees finding up to 4 errors. +}; + /** Encode a Bech32 or Bech32m string. If hrp contains uppercase characters, this will cause an * assertion error. Encoding must be one of BECH32 or BECH32M. */ std::string Encode(Encoding encoding, const std::string& hrp, const std::vector<uint8_t>& values); @@ -43,10 +51,10 @@ struct DecodeResult }; /** Decode a Bech32 or Bech32m string. */ -DecodeResult Decode(const std::string& str); +DecodeResult Decode(const std::string& str, CharLimit limit = CharLimit::BECH32); /** Return the positions of errors in a Bech32 string. */ -std::pair<std::string, std::vector<int>> LocateErrors(const std::string& str); +std::pair<std::string, std::vector<int>> LocateErrors(const std::string& str, CharLimit limit = CharLimit::BECH32); } // namespace bech32 diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index a13a693ad7..733f8085ca 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -18,6 +18,7 @@ #include <vector> using namespace std::chrono_literals; +using util::Join; const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index 8c421c3fec..a1b880e40b 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -16,6 +16,8 @@ #include <sstream> #include <vector> +using util::SplitString; + static const char* DEFAULT_BENCH_FILTER = ".*"; static constexpr int64_t DEFAULT_MIN_TIME_MS{10}; /** Priority level default value, run "all" priority levels */ diff --git a/src/bench/checkblockindex.cpp b/src/bench/checkblockindex.cpp new file mode 100644 index 0000000000..e8a848dbd4 --- /dev/null +++ b/src/bench/checkblockindex.cpp @@ -0,0 +1,20 @@ +// Copyright (c) 2023-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <test/util/setup_common.h> +#include <validation.h> + +static void CheckBlockIndex(benchmark::Bench& bench) +{ + auto testing_setup{MakeNoLogFileContext<TestChain100Setup>()}; + // Mine some more blocks + testing_setup->mineBlocks(1000); + bench.run([&] { + testing_setup->m_node.chainman->CheckBlockIndex(); + }); +} + + +BENCHMARK(CheckBlockIndex, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 249b76ee85..171c61c46f 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -71,15 +71,15 @@ static void CoinSelection(benchmark::Bench& bench) /*change_output_size=*/ 34, /*change_spend_size=*/ 148, /*min_change_target=*/ CHANGE_LOWER, - /*effective_feerate=*/ CFeeRate(0), - /*long_term_feerate=*/ CFeeRate(0), - /*discard_feerate=*/ CFeeRate(0), + /*effective_feerate=*/ CFeeRate(20'000), + /*long_term_feerate=*/ CFeeRate(10'000), + /*discard_feerate=*/ CFeeRate(3000), /*tx_noinputs_size=*/ 0, /*avoid_partial=*/ false, }; auto group = wallet::GroupOutputs(wallet, available_coins, coin_selection_params, {{filter_standard}})[filter_standard]; bench.run([&] { - auto result = AttemptSelection(wallet.chain(), 1003 * COIN, group, coin_selection_params, /*allow_mixed_output_types=*/true); + auto result = AttemptSelection(wallet.chain(), 1002.99 * COIN, group, coin_selection_params, /*allow_mixed_output_types=*/true); assert(result); assert(result->GetSelectedValue() == 1003 * COIN); assert(result->GetInputSet().size() == 2); diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index 4d2a6f0c2a..ecbdcd48bb 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -16,6 +16,7 @@ #include <kernel/checks.h> #include <kernel/context.h> #include <kernel/validation_cache_sizes.h> +#include <kernel/warning.h> #include <consensus/validation.h> #include <core_io.h> @@ -28,6 +29,7 @@ #include <util/fs.h> #include <util/signalinterrupt.h> #include <util/task_runner.h> +#include <util/translation.h> #include <validation.h> #include <validationinterface.h> @@ -86,9 +88,13 @@ int main(int argc, char* argv[]) { std::cout << "Progress: " << title.original << ", " << progress_percent << ", " << resume_possible << std::endl; } - void warning(const bilingual_str& warning) override + void warningSet(kernel::Warning id, const bilingual_str& message) override { - std::cout << "Warning: " << warning.original << std::endl; + std::cout << "Warning " << static_cast<int>(id) << " set: " << message.original << std::endl; + } + void warningUnset(kernel::Warning id) override + { + std::cout << "Warning " << static_cast<int>(id) << " unset" << std::endl; } void flushError(const bilingual_str& message) override { @@ -151,7 +157,7 @@ int main(int argc, char* argv[]) { LOCK(chainman.GetMutex()); std::cout - << "\t" << "Reindexing: " << std::boolalpha << chainman.m_blockman.m_reindexing.load() << std::noboolalpha << std::endl + << "\t" << "Blockfiles Indexed: " << std::boolalpha << chainman.m_blockman.m_blockfiles_indexed.load() << std::noboolalpha << std::endl << "\t" << "Snapshot Active: " << std::boolalpha << chainman.IsSnapshotActive() << std::noboolalpha << std::endl << "\t" << "Active Height: " << chainman.ActiveHeight() << std::endl << "\t" << "Active IBD: " << std::boolalpha << chainman.IsInitialBlockDownload() << std::noboolalpha << std::endl; diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index cc639d8793..44fc273163 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -42,6 +42,9 @@ #include <event2/keyvalq_struct.h> #include <support/events.h> +using util::Join; +using util::ToString; + // The server returns time values from a mockable system clock, but it is not // trivial to get the mocked time from the server, nor is it needed for now, so // just use a plain system_clock. @@ -298,7 +301,7 @@ public: } addresses.pushKV("total", total); result.pushKV("addresses_known", std::move(addresses)); - return JSONRPCReplyObj(std::move(result), NullUniValue, /*id=*/1, JSONRPCVersion::V1_LEGACY); + return JSONRPCReplyObj(std::move(result), NullUniValue, /*id=*/1, JSONRPCVersion::V2); } }; @@ -367,7 +370,7 @@ public: } result.pushKV("relayfee", batch[ID_NETWORKINFO]["result"]["relayfee"]); result.pushKV("warnings", batch[ID_NETWORKINFO]["result"]["warnings"]); - return JSONRPCReplyObj(std::move(result), NullUniValue, /*id=*/1, JSONRPCVersion::V1_LEGACY); + return JSONRPCReplyObj(std::move(result), NullUniValue, /*id=*/1, JSONRPCVersion::V2); } }; @@ -622,7 +625,7 @@ public: } } - return JSONRPCReplyObj(UniValue{result}, NullUniValue, /*id=*/1, JSONRPCVersion::V1_LEGACY); + return JSONRPCReplyObj(UniValue{result}, NullUniValue, /*id=*/1, JSONRPCVersion::V2); } const std::string m_help_doc{ @@ -709,7 +712,7 @@ public: UniValue result(UniValue::VOBJ); result.pushKV("address", address_str); result.pushKV("blocks", reply.get_obj()["result"]); - return JSONRPCReplyObj(std::move(result), NullUniValue, /*id=*/1, JSONRPCVersion::V1_LEGACY); + return JSONRPCReplyObj(std::move(result), NullUniValue, /*id=*/1, JSONRPCVersion::V2); } protected: std::string address_str; @@ -743,8 +746,41 @@ static UniValue CallRPC(BaseRequestHandler* rh, const std::string& strMethod, co // 2. port in -rpcconnect (ie following : in ipv4 or ]: in ipv6) // 3. default port for chain uint16_t port{BaseParams().RPCPort()}; - SplitHostPort(gArgs.GetArg("-rpcconnect", DEFAULT_RPCCONNECT), port, host); - port = static_cast<uint16_t>(gArgs.GetIntArg("-rpcport", port)); + { + uint16_t rpcconnect_port{0}; + const std::string rpcconnect_str = gArgs.GetArg("-rpcconnect", DEFAULT_RPCCONNECT); + if (!SplitHostPort(rpcconnect_str, rpcconnect_port, host)) { + // Uses argument provided as-is + // (rather than value parsed) + // to aid the user in troubleshooting + throw std::runtime_error(strprintf("Invalid port provided in -rpcconnect: %s", rpcconnect_str)); + } else { + if (rpcconnect_port != 0) { + // Use the valid port provided in rpcconnect + port = rpcconnect_port; + } // else, no port was provided in rpcconnect (continue using default one) + } + + if (std::optional<std::string> rpcport_arg = gArgs.GetArg("-rpcport")) { + // -rpcport was specified + const uint16_t rpcport_int{ToIntegral<uint16_t>(rpcport_arg.value()).value_or(0)}; + if (rpcport_int == 0) { + // Uses argument provided as-is + // (rather than value parsed) + // to aid the user in troubleshooting + throw std::runtime_error(strprintf("Invalid port provided in -rpcport: %s", rpcport_arg.value())); + } + + // Use the valid port provided + port = rpcport_int; + + // If there was a valid port provided in rpcconnect, + // rpcconnect_port is non-zero. + if (rpcconnect_port != 0) { + tfm::format(std::cerr, "Warning: Port specified in both -rpcconnect and -rpcport. Using -rpcport %u\n", port); + } + } + } // Obtain event base raii_event_base base = obtain_event_base(); diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index cfac50e090..89faa0123a 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -32,6 +32,11 @@ #include <functional> #include <memory> +using util::SplitString; +using util::ToString; +using util::TrimString; +using util::TrimStringView; + static bool fCreateBlank; static std::map<std::string,UniValue> registers; static const int CONTINUE_EXECUTION=-1; @@ -203,12 +208,12 @@ static CAmount ExtractAndValidateValue(const std::string& strValue) static void MutateTxVersion(CMutableTransaction& tx, const std::string& cmdVal) { - int64_t newVersion; - if (!ParseInt64(cmdVal, &newVersion) || newVersion < 1 || newVersion > TX_MAX_STANDARD_VERSION) { + uint32_t newVersion; + if (!ParseUInt32(cmdVal, &newVersion) || newVersion < 1 || newVersion > TX_MAX_STANDARD_VERSION) { throw std::runtime_error("Invalid TX version requested: '" + cmdVal + "'"); } - tx.nVersion = (int) newVersion; + tx.version = newVersion; } static void MutateTxLocktime(CMutableTransaction& tx, const std::string& cmdVal) diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 87af347473..b6f5c3f15d 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -24,6 +24,8 @@ #include <string> #include <tuple> +using util::Join; + const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; static void SetupWalletToolArgs(ArgsManager& argsman) diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 0b89aa42af..a09bb5c9da 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -17,6 +17,7 @@ #include <kernel/context.h> #include <node/context.h> #include <node/interface_ui.h> +#include <node/warnings.h> #include <noui.h> #include <util/check.h> #include <util/exception.h> @@ -181,6 +182,8 @@ static bool AppInit(NodeContext& node) return false; } + node.warnings = std::make_unique<node::Warnings>(); + node.kernel = std::make_unique<kernel::Context>(); node.ecc_context = std::make_unique<ECC_Context>(); if (!AppInitSanityChecks(*node.kernel)) diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index e045b88513..5e6702ccc3 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -16,6 +16,8 @@ #include <util/golombrice.h> #include <util/string.h> +using util::Join; + static const std::map<BlockFilterType, std::string> g_filter_types = { {BlockFilterType::BASIC, "basic"}, }; diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 539578085b..5d4401b719 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -21,6 +21,8 @@ #include <stdexcept> #include <vector> +using util::SplitString; + void ReadSigNetArgs(const ArgsManager& args, CChainParams::SigNetOptions& options) { if (args.IsArgSet("-signetseednode")) { diff --git a/src/clientversion.cpp b/src/clientversion.cpp index 6b9727a158..e52703c8bf 100644 --- a/src/clientversion.cpp +++ b/src/clientversion.cpp @@ -13,6 +13,8 @@ #include <string> #include <vector> +using util::Join; + /** * Name of client reported in the 'version' message. Report the same name * for both bitcoind and bitcoin-qt, to make it harder for attackers to diff --git a/src/coins.cpp b/src/coins.cpp index b62653e0de..a4e4d4ad32 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -361,7 +361,7 @@ static bool ExecuteBackedWrapper(Func func, const std::vector<std::function<void for (const auto& f : err_callbacks) { f(); } - LogPrintf("Error reading from database: %s\n", e.what()); + LogError("Error reading from database: %s\n", e.what()); // Starting the shutdown sequence and returning false to the caller would be // interpreted as 'entry not found' (as opposed to unable to read data), and // could lead to invalid interpretation. Just exit immediately, as we can't diff --git a/src/common/config.cpp b/src/common/config.cpp index 1c85273f69..98223fc3e3 100644 --- a/src/common/config.cpp +++ b/src/common/config.cpp @@ -27,6 +27,9 @@ #include <utility> #include <vector> +using util::TrimString; +using util::TrimStringView; + static bool GetConfigOptions(std::istream& stream, const std::string& filepath, std::string& error, std::vector<std::pair<std::string, std::string>>& options, std::list<SectionInfo>& sections) { std::string str, prefix; diff --git a/src/common/messages.cpp b/src/common/messages.cpp new file mode 100644 index 0000000000..9e88ca8b0f --- /dev/null +++ b/src/common/messages.cpp @@ -0,0 +1,139 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <common/messages.h> + +#include <common/types.h> +#include <policy/fees.h> +#include <node/types.h> +#include <tinyformat.h> +#include <util/strencodings.h> +#include <util/string.h> +#include <util/translation.h> + +#include <cassert> +#include <map> +#include <string> +#include <utility> +#include <vector> + +using node::TransactionError; +using util::Join; + +namespace common { +std::string StringForFeeReason(FeeReason reason) +{ + static const std::map<FeeReason, std::string> fee_reason_strings = { + {FeeReason::NONE, "None"}, + {FeeReason::HALF_ESTIMATE, "Half Target 60% Threshold"}, + {FeeReason::FULL_ESTIMATE, "Target 85% Threshold"}, + {FeeReason::DOUBLE_ESTIMATE, "Double Target 95% Threshold"}, + {FeeReason::CONSERVATIVE, "Conservative Double Target longer horizon"}, + {FeeReason::MEMPOOL_MIN, "Mempool Min Fee"}, + {FeeReason::PAYTXFEE, "PayTxFee set"}, + {FeeReason::FALLBACK, "Fallback fee"}, + {FeeReason::REQUIRED, "Minimum Required Fee"}, + }; + auto reason_string = fee_reason_strings.find(reason); + + if (reason_string == fee_reason_strings.end()) return "Unknown"; + + return reason_string->second; +} + +const std::vector<std::pair<std::string, FeeEstimateMode>>& FeeModeMap() +{ + static const std::vector<std::pair<std::string, FeeEstimateMode>> FEE_MODES = { + {"unset", FeeEstimateMode::UNSET}, + {"economical", FeeEstimateMode::ECONOMICAL}, + {"conservative", FeeEstimateMode::CONSERVATIVE}, + }; + return FEE_MODES; +} + +std::string FeeModes(const std::string& delimiter) +{ + return Join(FeeModeMap(), delimiter, [&](const std::pair<std::string, FeeEstimateMode>& i) { return i.first; }); +} + +std::string InvalidEstimateModeErrorMessage() +{ + return "Invalid estimate_mode parameter, must be one of: \"" + FeeModes("\", \"") + "\""; +} + +bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode) +{ + auto searchkey = ToUpper(mode_string); + for (const auto& pair : FeeModeMap()) { + if (ToUpper(pair.first) == searchkey) { + fee_estimate_mode = pair.second; + return true; + } + } + return false; +} + +bilingual_str PSBTErrorString(PSBTError err) +{ + switch (err) { + case PSBTError::MISSING_INPUTS: + return Untranslated("Inputs missing or spent"); + case PSBTError::SIGHASH_MISMATCH: + return Untranslated("Specified sighash value does not match value stored in PSBT"); + case PSBTError::EXTERNAL_SIGNER_NOT_FOUND: + return Untranslated("External signer not found"); + case PSBTError::EXTERNAL_SIGNER_FAILED: + return Untranslated("External signer failed to sign"); + case PSBTError::UNSUPPORTED: + return Untranslated("Signer does not support PSBT"); + // no default case, so the compiler can warn about missing cases + } + assert(false); +} + +bilingual_str TransactionErrorString(const TransactionError err) +{ + switch (err) { + case TransactionError::OK: + return Untranslated("No error"); + case TransactionError::MISSING_INPUTS: + return Untranslated("Inputs missing or spent"); + case TransactionError::ALREADY_IN_CHAIN: + return Untranslated("Transaction already in block chain"); + case TransactionError::MEMPOOL_REJECTED: + return Untranslated("Transaction rejected by mempool"); + case TransactionError::MEMPOOL_ERROR: + return Untranslated("Mempool internal error"); + case TransactionError::MAX_FEE_EXCEEDED: + return Untranslated("Fee exceeds maximum configured by user (e.g. -maxtxfee, maxfeerate)"); + case TransactionError::MAX_BURN_EXCEEDED: + return Untranslated("Unspendable output exceeds maximum configured by user (maxburnamount)"); + case TransactionError::INVALID_PACKAGE: + return Untranslated("Transaction rejected due to invalid package"); + // no default case, so the compiler can warn about missing cases + } + assert(false); +} + +bilingual_str ResolveErrMsg(const std::string& optname, const std::string& strBind) +{ + return strprintf(_("Cannot resolve -%s address: '%s'"), optname, strBind); +} + +bilingual_str InvalidPortErrMsg(const std::string& optname, const std::string& invalid_value) +{ + return strprintf(_("Invalid port specified in %s: '%s'"), optname, invalid_value); +} + +bilingual_str AmountHighWarn(const std::string& optname) +{ + return strprintf(_("%s is set very high!"), optname); +} + +bilingual_str AmountErrMsg(const std::string& optname, const std::string& strValue) +{ + return strprintf(_("Invalid amount for -%s=<amount>: '%s'"), optname, strValue); +} +} // namespace common diff --git a/src/common/messages.h b/src/common/messages.h new file mode 100644 index 0000000000..68e7bb2169 --- /dev/null +++ b/src/common/messages.h @@ -0,0 +1,38 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +//! @file common/messages.h is a home for simple string functions returning +//! descriptive messages that are used in RPC and GUI interfaces or log +//! messages, and are called in different parts of the codebase across +//! node/wallet/gui boundaries. + +#ifndef BITCOIN_COMMON_MESSAGES_H +#define BITCOIN_COMMON_MESSAGES_H + +#include <string> + +struct bilingual_str; + +enum class FeeEstimateMode; +enum class FeeReason; +namespace node { +enum class TransactionError; +} // namespace node + +namespace common { +enum class PSBTError; +bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode); +std::string StringForFeeReason(FeeReason reason); +std::string FeeModes(const std::string& delimiter); +std::string InvalidEstimateModeErrorMessage(); +bilingual_str PSBTErrorString(PSBTError error); +bilingual_str TransactionErrorString(const node::TransactionError error); +bilingual_str ResolveErrMsg(const std::string& optname, const std::string& strBind); +bilingual_str InvalidPortErrMsg(const std::string& optname, const std::string& strPort); +bilingual_str AmountHighWarn(const std::string& optname); +bilingual_str AmountErrMsg(const std::string& optname, const std::string& strValue); +} // namespace common + +#endif // BITCOIN_COMMON_MESSAGES_H diff --git a/src/util/message.cpp b/src/common/signmessage.cpp index 1afb28cc10..1612751e44 100644 --- a/src/util/message.cpp +++ b/src/common/signmessage.cpp @@ -3,12 +3,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/signmessage.h> #include <hash.h> #include <key.h> #include <key_io.h> #include <pubkey.h> #include <uint256.h> -#include <util/message.h> #include <util/strencodings.h> #include <cassert> diff --git a/src/util/message.h b/src/common/signmessage.h index d0e2422574..215b563bbb 100644 --- a/src/util/message.h +++ b/src/common/signmessage.h @@ -3,8 +3,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_UTIL_MESSAGE_H -#define BITCOIN_UTIL_MESSAGE_H +#ifndef BITCOIN_COMMON_SIGNMESSAGE_H +#define BITCOIN_COMMON_SIGNMESSAGE_H #include <uint256.h> @@ -74,4 +74,4 @@ uint256 MessageHash(const std::string& message); std::string SigningResultString(const SigningResult res); -#endif // BITCOIN_UTIL_MESSAGE_H +#endif // BITCOIN_COMMON_SIGNMESSAGE_H diff --git a/src/common/system.cpp b/src/common/system.cpp index ddd0feda3b..6d04c8a7bc 100644 --- a/src/common/system.cpp +++ b/src/common/system.cpp @@ -28,6 +28,8 @@ #include <string> #include <thread> +using util::ReplaceAll; + // Application startup time (used for uptime calculation) const int64_t nStartupTime = GetTime(); diff --git a/src/common/types.h b/src/common/types.h new file mode 100644 index 0000000000..0d9cb67ce9 --- /dev/null +++ b/src/common/types.h @@ -0,0 +1,26 @@ +// Copyright (c) 2010-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +//! @file common/types.h is a home for simple enum and struct type definitions +//! that can be used internally by functions in the libbitcoin_common library, +//! but also used externally by node, wallet, and GUI code. +//! +//! This file is intended to define only simple types that do not have external +//! dependencies. More complicated types should be defined in dedicated header +//! files. + +#ifndef BITCOIN_COMMON_TYPES_H +#define BITCOIN_COMMON_TYPES_H + +namespace common { +enum class PSBTError { + MISSING_INPUTS, + SIGHASH_MISMATCH, + EXTERNAL_SIGNER_NOT_FOUND, + EXTERNAL_SIGNER_FAILED, + UNSUPPORTED, +}; +} // namespace common + +#endif // BITCOIN_COMMON_TYPES_H diff --git a/src/compressor.h b/src/compressor.h index 0968454679..a0970c595e 100644 --- a/src/compressor.h +++ b/src/compressor.h @@ -55,8 +55,8 @@ struct ScriptCompression { /** * make this static for now (there are only 6 special scripts defined) - * this can potentially be extended together with a new nVersion for - * transactions, in which case this value becomes dependent on nVersion + * this can potentially be extended together with a new version for + * transactions, in which case this value becomes dependent on version * and nHeight of the enclosing transaction. */ static const unsigned int nSpecialScripts = 6; diff --git a/src/consensus/tx_verify.cpp b/src/consensus/tx_verify.cpp index 154146f08d..95466b759c 100644 --- a/src/consensus/tx_verify.cpp +++ b/src/consensus/tx_verify.cpp @@ -48,11 +48,7 @@ std::pair<int, int64_t> CalculateSequenceLocks(const CTransaction &tx, int flags int nMinHeight = -1; int64_t nMinTime = -1; - // tx.nVersion is signed integer so requires cast to unsigned otherwise - // we would be doing a signed comparison and half the range of nVersion - // wouldn't support BIP 68. - bool fEnforceBIP68 = static_cast<uint32_t>(tx.nVersion) >= 2 - && flags & LOCKTIME_VERIFY_SEQUENCE; + bool fEnforceBIP68 = tx.version >= 2 && flags & LOCKTIME_VERIFY_SEQUENCE; // Do not enforce sequence numbers as a relative lock time // unless we have been instructed to diff --git a/src/core_read.cpp b/src/core_read.cpp index 5956d9df5f..114f92fc45 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -16,6 +16,8 @@ #include <algorithm> #include <string> +using util::SplitString; + namespace { class OpCodeParser { diff --git a/src/core_write.cpp b/src/core_write.cpp index 3a2bf865fc..253dfde100 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -174,9 +174,7 @@ void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry entry.pushKV("txid", tx.GetHash().GetHex()); entry.pushKV("hash", tx.GetWitnessHash().GetHex()); - // Transaction version is actually unsigned in consensus checks, just signed in memory, - // so cast to unsigned before giving it to the user. - entry.pushKV("version", static_cast<int64_t>(static_cast<uint32_t>(tx.nVersion))); + entry.pushKV("version", tx.version); entry.pushKV("size", tx.GetTotalSize()); entry.pushKV("vsize", (GetTransactionWeight(tx) + WITNESS_SCALE_FACTOR - 1) / WITNESS_SCALE_FACTOR); entry.pushKV("weight", GetTransactionWeight(tx)); diff --git a/src/crypto/hex_base.cpp b/src/crypto/hex_base.cpp new file mode 100644 index 0000000000..67d691b63e --- /dev/null +++ b/src/crypto/hex_base.cpp @@ -0,0 +1,67 @@ +// Copyright (c) 2009-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <crypto/hex_base.h> + +#include <array> +#include <cstring> +#include <string> + +namespace { + +using ByteAsHex = std::array<char, 2>; + +constexpr std::array<ByteAsHex, 256> CreateByteToHexMap() +{ + constexpr char hexmap[16] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + + std::array<ByteAsHex, 256> byte_to_hex{}; + for (size_t i = 0; i < byte_to_hex.size(); ++i) { + byte_to_hex[i][0] = hexmap[i >> 4]; + byte_to_hex[i][1] = hexmap[i & 15]; + } + return byte_to_hex; +} + +} // namespace + +std::string HexStr(const Span<const uint8_t> s) +{ + std::string rv(s.size() * 2, '\0'); + static constexpr auto byte_to_hex = CreateByteToHexMap(); + static_assert(sizeof(byte_to_hex) == 512); + + char* it = rv.data(); + for (uint8_t v : s) { + std::memcpy(it, byte_to_hex[v].data(), 2); + it += 2; + } + + assert(it == rv.data() + rv.size()); + return rv; +} + +const signed char p_util_hexdigit[256] = +{ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + 0,1,2,3,4,5,6,7,8,9,-1,-1,-1,-1,-1,-1, + -1,0xa,0xb,0xc,0xd,0xe,0xf,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,0xa,0xb,0xc,0xd,0xe,0xf,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, }; + +signed char HexDigit(char c) +{ + return p_util_hexdigit[(unsigned char)c]; +} + diff --git a/src/crypto/hex_base.h b/src/crypto/hex_base.h new file mode 100644 index 0000000000..cdfea68c29 --- /dev/null +++ b/src/crypto/hex_base.h @@ -0,0 +1,23 @@ +// Copyright (c) 2009-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_HEX_BASE_H +#define BITCOIN_CRYPTO_HEX_BASE_H + +#include <span.h> + +#include <cstddef> +#include <cstdint> +#include <string> + +/** + * Convert a span of bytes to a lower-case hexadecimal string. + */ +std::string HexStr(const Span<const uint8_t> s); +inline std::string HexStr(const Span<const char> s) { return HexStr(MakeUCharSpan(s)); } +inline std::string HexStr(const Span<const std::byte> s) { return HexStr(MakeUCharSpan(s)); } + +signed char HexDigit(char c); + +#endif // BITCOIN_CRYPTO_HEX_BASE_H diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 3eb34dbe6a..128597157d 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -23,6 +23,9 @@ #include <string> #include <vector> +using util::SplitString; +using util::TrimStringView; + /** WWW-Authenticate to present with 401 Unauthorized response */ static const char* WWW_AUTH_HEADER_DATA = "Basic realm=\"jsonrpc\""; diff --git a/src/i2p.cpp b/src/i2p.cpp index 962adb124d..a907cfeacb 100644 --- a/src/i2p.cpp +++ b/src/i2p.cpp @@ -12,12 +12,12 @@ #include <netaddress.h> #include <netbase.h> #include <random.h> +#include <script/parsing.h> #include <sync.h> #include <tinyformat.h> #include <util/fs.h> #include <util/readwritefile.h> #include <util/sock.h> -#include <util/spanparsing.h> #include <util/strencodings.h> #include <util/threadinterrupt.h> @@ -26,6 +26,8 @@ #include <stdexcept> #include <string> +using util::Split; + namespace i2p { /** @@ -308,7 +310,7 @@ Session::Reply Session::SendRequestAndGetReply(const Sock& sock, reply.full = sock.RecvUntilTerminator('\n', recv_timeout, *m_interrupt, MAX_MSG_SIZE); - for (const auto& kv : spanparsing::Split(reply.full, ' ')) { + for (const auto& kv : Split(reply.full, ' ')) { const auto& pos = std::find(kv.begin(), kv.end(), '='); if (pos != kv.end()) { reply.keys.emplace(std::string{kv.begin(), pos}, std::string{pos + 1, kv.end()}); diff --git a/src/index/base.cpp b/src/index/base.cpp index e66c89f9e4..955d7b67c9 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -17,7 +17,6 @@ #include <util/thread.h> #include <util/translation.h> #include <validation.h> // For g_chainman -#include <warnings.h> #include <string> #include <utility> @@ -31,7 +30,7 @@ template <typename... Args> void BaseIndex::FatalErrorf(const char* fmt, const Args&... args) { auto message = tfm::format(fmt, args...); - node::AbortNode(m_chain->context()->shutdown, m_chain->context()->exit_status, Untranslated(message)); + node::AbortNode(m_chain->context()->shutdown, m_chain->context()->exit_status, Untranslated(message), m_chain->context()->warnings.get()); } CBlockLocator GetLocator(interfaces::Chain& chain, const uint256& block_hash) diff --git a/src/init.cpp b/src/init.cpp index 0aac2ac65f..5bb82dc320 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -115,6 +115,9 @@ #include <zmq/zmqrpc.h> #endif +using common::AmountErrMsg; +using common::InvalidPortErrMsg; +using common::ResolveErrMsg; using kernel::DumpMempool; using kernel::LoadMempool; using kernel::ValidationCacheSizes; @@ -133,6 +136,9 @@ using node::NodeContext; using node::ShouldPersistMempool; using node::ImportBlocks; using node::VerifyLoadedChainstate; +using util::Join; +using util::ReplaceAll; +using util::ToString; static constexpr bool DEFAULT_PROXYRANDOMIZE{true}; static constexpr bool DEFAULT_REST_ENABLE{false}; @@ -403,7 +409,7 @@ static void HandleSIGHUP(int) static BOOL WINAPI consoleCtrlHandler(DWORD dwCtrlType) { if (!(*Assert(g_shutdown))()) { - LogPrintf("Error: failed to send shutdown signal on Ctrl-C\n"); + LogError("Failed to send shutdown signal on Ctrl-C\n"); return false; } Sleep(INFINITE); @@ -598,7 +604,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-checkblocks=<n>", strprintf("How many blocks to check at startup (default: %u, 0 = all)", DEFAULT_CHECKBLOCKS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checklevel=<n>", strprintf("How thorough the block verification of -checkblocks is: %s (0-4, default: %u)", Join(CHECKLEVEL_DOC, ", "), DEFAULT_CHECKLEVEL), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-checkblockindex", strprintf("Do a consistency check for the block tree, chainstate, and other validation data structures occasionally. (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-checkblockindex", strprintf("Do a consistency check for the block tree, chainstate, and other validation data structures every <n> operations. Use 0 to disable. (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checkaddrman=<n>", strprintf("Run addrman consistency checks every <n> operations. Use 0 to disable. (default: %u)", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checkmempool=<n>", strprintf("Run mempool consistency checks every <n> transactions. Use 0 to disable. (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checkpoints", strprintf("Enable rejection of any forks from the known historical chain until block %s (default: %u)", defaultChainParams->Checkpoints().GetHeight(), DEFAULT_CHECKPOINTS_ENABLED), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); @@ -834,7 +840,7 @@ std::set<BlockFilterType> g_enabled_filter_types; // Since LogPrintf may itself allocate memory, set the handler directly // to terminate first. std::set_new_handler(std::terminate); - LogPrintf("Error: Out of memory. Terminating.\n"); + LogError("Out of memory. Terminating.\n"); // The log was successful, terminate now. std::terminate(); @@ -1169,9 +1175,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) scheduler.scheduleEvery([&args, &node]{ constexpr uint64_t min_disk_space = 50 << 20; // 50 MB if (!CheckDiskSpace(args.GetBlocksDirPath(), min_disk_space)) { - LogPrintf("Shutting down due to lack of disk space!\n"); + LogError("Shutting down due to lack of disk space!\n"); if (!(*Assert(node.shutdown))()) { - LogPrintf("Error: failed to send shutdown signal after disk space check\n"); + LogError("Failed to send shutdown signal after disk space check\n"); } } }, std::chrono::minutes{5}); @@ -1480,9 +1486,8 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // ********************************************************* Step 7: load block chain - node.notifications = std::make_unique<KernelNotifications>(*Assert(node.shutdown), node.exit_status); + node.notifications = std::make_unique<KernelNotifications>(*Assert(node.shutdown), node.exit_status, *Assert(node.warnings)); ReadNotificationArgs(args, *node.notifications); - bool fReindexChainState = args.GetBoolArg("-reindex-chainstate", false); ChainstateManager::Options chainman_opts{ .chainparams = chainparams, .datadir = args.GetDataDirNet(), @@ -1523,16 +1528,17 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) if (!result) { return InitError(util::ErrorString(result)); } - mempool_opts.check_ratio = std::clamp<int>(mempool_opts.check_ratio, 0, 1'000'000); - int64_t descendant_limit_bytes = mempool_opts.limits.descendant_size_vbytes * 40; - if (mempool_opts.max_size_bytes < 0 || mempool_opts.max_size_bytes < descendant_limit_bytes) { - return InitError(strprintf(_("-maxmempool must be at least %d MB"), std::ceil(descendant_limit_bytes / 1'000'000.0))); - } - LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", cache_sizes.coins * (1.0 / 1024 / 1024), mempool_opts.max_size_bytes * (1.0 / 1024 / 1024)); + bool do_reindex{args.GetBoolArg("-reindex", false)}; + const bool do_reindex_chainstate{args.GetBoolArg("-reindex-chainstate", false)}; for (bool fLoaded = false; !fLoaded && !ShutdownRequested(node);) { - node.mempool = std::make_unique<CTxMemPool>(mempool_opts); + bilingual_str mempool_error; + node.mempool = std::make_unique<CTxMemPool>(mempool_opts, mempool_error); + if (!mempool_error.empty()) { + return InitError(mempool_error); + } + LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", cache_sizes.coins * (1.0 / 1024 / 1024), mempool_opts.max_size_bytes * (1.0 / 1024 / 1024)); node.chainman = std::make_unique<ChainstateManager>(*Assert(node.shutdown), chainman_opts, blockman_opts); ChainstateManager& chainman = *node.chainman; @@ -1558,8 +1564,8 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) node::ChainstateLoadOptions options; options.mempool = Assert(node.mempool.get()); - options.reindex = chainman.m_blockman.m_reindexing; - options.reindex_chainstate = fReindexChainState; + options.wipe_block_tree_db = do_reindex; + options.wipe_chainstate_db = do_reindex || do_reindex_chainstate; options.prune = chainman.m_blockman.IsPruneMode(); options.check_blocks = args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); options.check_level = args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL); @@ -1576,7 +1582,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) try { return f(); } catch (const std::exception& e) { - LogPrintf("%s\n", e.what()); + LogError("%s\n", e.what()); return std::make_tuple(node::ChainstateLoadStatus::FAILURE, _("Error opening block database")); } }; @@ -1600,18 +1606,18 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) if (!fLoaded && !ShutdownRequested(node)) { // first suggest a reindex - if (!options.reindex) { + if (!do_reindex) { bool fRet = uiInterface.ThreadSafeQuestion( error + Untranslated(".\n\n") + _("Do you want to rebuild the block database now?"), error.original + ".\nPlease restart with -reindex or -reindex-chainstate to recover.", "", CClientUIInterface::MSG_ERROR | CClientUIInterface::BTN_ABORT); if (fRet) { - chainman.m_blockman.m_reindexing = true; + do_reindex = true; if (!Assert(node.shutdown)->reset()) { - LogPrintf("Internal error: failed to reset shutdown signal.\n"); + LogError("Internal error: failed to reset shutdown signal.\n"); } } else { - LogPrintf("Aborted block database rebuild. Exiting.\n"); + LogError("Aborted block database rebuild. Exiting.\n"); return false; } } else { @@ -1633,23 +1639,24 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) assert(!node.peerman); node.peerman = PeerManager::make(*node.connman, *node.addrman, node.banman.get(), chainman, - *node.mempool, peerman_opts); + *node.mempool, *node.warnings, + peerman_opts); validation_signals.RegisterValidationInterface(node.peerman.get()); // ********************************************************* Step 8: start indexers if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { - g_txindex = std::make_unique<TxIndex>(interfaces::MakeChain(node), cache_sizes.tx_index, false, chainman.m_blockman.m_reindexing); + g_txindex = std::make_unique<TxIndex>(interfaces::MakeChain(node), cache_sizes.tx_index, false, do_reindex); node.indexes.emplace_back(g_txindex.get()); } for (const auto& filter_type : g_enabled_filter_types) { - InitBlockFilterIndex([&]{ return interfaces::MakeChain(node); }, filter_type, cache_sizes.filter_index, false, chainman.m_blockman.m_reindexing); + InitBlockFilterIndex([&]{ return interfaces::MakeChain(node); }, filter_type, cache_sizes.filter_index, false, do_reindex); node.indexes.emplace_back(GetBlockFilterIndex(filter_type)); } if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) { - g_coin_stats_index = std::make_unique<CoinStatsIndex>(interfaces::MakeChain(node), /*cache_size=*/0, false, chainman.m_blockman.m_reindexing); + g_coin_stats_index = std::make_unique<CoinStatsIndex>(interfaces::MakeChain(node), /*cache_size=*/0, false, do_reindex); node.indexes.emplace_back(g_coin_stats_index.get()); } @@ -1668,7 +1675,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // if pruning, perform the initial blockstore prune // after any wallet rescanning has taken place. if (chainman.m_blockman.IsPruneMode()) { - if (!chainman.m_blockman.m_reindexing) { + if (chainman.m_blockman.m_blockfiles_indexed) { LOCK(cs_main); for (Chainstate* chainstate : chainman.GetAll()) { uiInterface.InitMessage(_("Pruning blockstore…").translated); @@ -1694,7 +1701,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) int chain_active_height = WITH_LOCK(cs_main, return chainman.ActiveChain().Height()); // On first startup, warn on low block storage space - if (!chainman.m_blockman.m_reindexing && !fReindexChainState && chain_active_height <= 1) { + if (!do_reindex && !do_reindex_chainstate && chain_active_height <= 1) { uint64_t assumed_chain_bytes{chainparams.AssumedBlockchainSize() * 1024 * 1024 * 1024}; uint64_t additional_bytes_needed{ chainman.m_blockman.IsPruneMode() ? @@ -1746,7 +1753,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) if (args.GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) { LogPrintf("Stopping after block import\n"); if (!(*Assert(node.shutdown))()) { - LogPrintf("Error: failed to send shutdown signal after finishing block import\n"); + LogError("Failed to send shutdown signal after finishing block import\n"); } return; } diff --git a/src/init/common.cpp b/src/init/common.cpp index f473ee6d66..00ef879dc1 100644 --- a/src/init/common.cpp +++ b/src/init/common.cpp @@ -20,6 +20,8 @@ #include <string> #include <vector> +using util::SplitString; + namespace init { void AddLoggingArgs(ArgsManager& argsman) { diff --git a/src/interfaces/node.h b/src/interfaces/node.h index aeb2612c07..2bb895dd47 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -30,10 +30,10 @@ class RPCTimerInterface; class UniValue; class Proxy; enum class SynchronizationState; -enum class TransactionError; struct CNodeStateStats; struct bilingual_str; namespace node { +enum class TransactionError; struct NodeContext; } // namespace node namespace wallet { @@ -208,7 +208,7 @@ public: virtual std::optional<Coin> getUnspentOutput(const COutPoint& output) = 0; //! Broadcast transaction. - virtual TransactionError broadcastTransaction(CTransactionRef tx, CAmount max_tx_fee, std::string& err_string) = 0; + virtual node::TransactionError broadcastTransaction(CTransactionRef tx, CAmount max_tx_fee, std::string& err_string) = 0; //! Get wallet loader. virtual WalletLoader& walletLoader() = 0; diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index c41f35829d..f7bcca58cf 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -6,13 +6,13 @@ #define BITCOIN_INTERFACES_WALLET_H #include <addresstype.h> +#include <common/signmessage.h> #include <consensus/amount.h> #include <interfaces/chain.h> #include <pubkey.h> #include <script/script.h> #include <support/allocators/secure.h> #include <util/fs.h> -#include <util/message.h> #include <util/result.h> #include <util/ui_change_type.h> @@ -30,9 +30,14 @@ class CFeeRate; class CKey; enum class FeeReason; enum class OutputType; -enum class TransactionError; struct PartiallySignedTransaction; struct bilingual_str; +namespace common { +enum class PSBTError; +} // namespace common +namespace node { +enum class TransactionError; +} // namespace node namespace wallet { class CCoinControl; class CWallet; @@ -202,7 +207,7 @@ public: int& num_blocks) = 0; //! Fill PSBT. - virtual TransactionError fillPSBT(int sighash_type, + virtual std::optional<common::PSBTError> fillPSBT(int sighash_type, bool sign, bool bip32derivs, size_t* n_signed, diff --git a/src/kernel/blockmanager_opts.h b/src/kernel/blockmanager_opts.h index 16072b669b..deeba7e318 100644 --- a/src/kernel/blockmanager_opts.h +++ b/src/kernel/blockmanager_opts.h @@ -24,7 +24,6 @@ struct BlockManagerOpts { bool fast_prune{false}; const fs::path blocks_dir; Notifications& notifications; - bool reindex{false}; }; } // namespace kernel diff --git a/src/kernel/chainparams.cpp b/src/kernel/chainparams.cpp index de142cd2c9..af02c6963b 100644 --- a/src/kernel/chainparams.cpp +++ b/src/kernel/chainparams.cpp @@ -29,7 +29,7 @@ static CBlock CreateGenesisBlock(const char* pszTimestamp, const CScript& genesisOutputScript, uint32_t nTime, uint32_t nNonce, uint32_t nBits, int32_t nVersion, const CAmount& genesisReward) { CMutableTransaction txNew; - txNew.nVersion = 1; + txNew.version = 1; txNew.vin.resize(1); txNew.vout.resize(1); txNew.vin[0].scriptSig = CScript() << 486604799 << CScriptNum(4) << std::vector<unsigned char>((const unsigned char*)pszTimestamp, (const unsigned char*)pszTimestamp + strlen(pszTimestamp)); @@ -554,7 +554,7 @@ std::vector<int> CChainParams::GetAvailableSnapshotHeights() const return heights; } -std::optional<ChainType> GetNetworkForMagic(MessageStartChars& message) +std::optional<ChainType> GetNetworkForMagic(const MessageStartChars& message) { const auto mainnet_msg = CChainParams::Main()->MessageStart(); const auto testnet_msg = CChainParams::TestNet()->MessageStart(); diff --git a/src/kernel/chainparams.h b/src/kernel/chainparams.h index f396c1b42c..05ebd07ec7 100644 --- a/src/kernel/chainparams.h +++ b/src/kernel/chainparams.h @@ -184,6 +184,6 @@ protected: ChainTxData chainTxData; }; -std::optional<ChainType> GetNetworkForMagic(MessageStartChars& pchMessageStart); +std::optional<ChainType> GetNetworkForMagic(const MessageStartChars& pchMessageStart); #endif // BITCOIN_KERNEL_CHAINPARAMS_H diff --git a/src/kernel/chainstatemanager_opts.h b/src/kernel/chainstatemanager_opts.h index de5f78494a..076841c3c9 100644 --- a/src/kernel/chainstatemanager_opts.h +++ b/src/kernel/chainstatemanager_opts.h @@ -33,7 +33,7 @@ namespace kernel { struct ChainstateManagerOpts { const CChainParams& chainparams; fs::path datadir; - std::optional<bool> check_block_index{}; + std::optional<int32_t> check_block_index{}; bool checkpoints_enabled{DEFAULT_CHECKPOINTS_ENABLED}; //! If set, it will override the minimum work we will assume exists on some valid chain. std::optional<arith_uint256> minimum_chain_work{}; diff --git a/src/kernel/notifications_interface.h b/src/kernel/notifications_interface.h index 7283a88e86..8e090dd7db 100644 --- a/src/kernel/notifications_interface.h +++ b/src/kernel/notifications_interface.h @@ -16,6 +16,8 @@ namespace kernel { //! Result type for use with std::variant to indicate that an operation should be interrupted. struct Interrupted{}; +enum class Warning; + //! Simple result type for functions that need to propagate an interrupt status and don't have other return values. using InterruptResult = std::variant<std::monostate, Interrupted>; @@ -38,7 +40,8 @@ public: [[nodiscard]] virtual InterruptResult blockTip(SynchronizationState state, CBlockIndex& index) { return {}; } virtual void headerTip(SynchronizationState state, int64_t height, int64_t timestamp, bool presync) {} virtual void progress(const bilingual_str& title, int progress_percent, bool resume_possible) {} - virtual void warning(const bilingual_str& warning) {} + virtual void warningSet(Warning id, const bilingual_str& message) {} + virtual void warningUnset(Warning id) {} //! The flush error notification is sent to notify the user that an error //! occurred while flushing block data to disk. Kernel code may ignore flush diff --git a/src/kernel/warning.h b/src/kernel/warning.h new file mode 100644 index 0000000000..453f36c552 --- /dev/null +++ b/src/kernel/warning.h @@ -0,0 +1,14 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_KERNEL_WARNING_H +#define BITCOIN_KERNEL_WARNING_H + +namespace kernel { +enum class Warning { + UNKNOWN_NEW_RULES_ACTIVATED, + LARGE_WORK_INVALID_CHAIN, +}; +} // namespace kernel +#endif // BITCOIN_KERNEL_WARNING_H diff --git a/src/key.cpp b/src/key.cpp index e8458f2e3b..97d7821e74 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -166,12 +166,6 @@ void CKey::MakeNewKey(bool fCompressedIn) { fCompressed = fCompressedIn; } -bool CKey::Negate() -{ - assert(keydata); - return secp256k1_ec_seckey_negate(secp256k1_context_sign, keydata->data()); -} - CPrivKey CKey::GetPrivKey() const { assert(keydata); CPrivKey seckey; @@ -124,9 +124,6 @@ public: //! Generate a new private key using a cryptographic PRNG. void MakeNewKey(bool fCompressed); - //! Negate private key - bool Negate(); - /** * Convert the private key to a CPrivKey (serialized OpenSSL private key data). * This is expensive. diff --git a/src/leveldb/include/leveldb/status.h b/src/leveldb/include/leveldb/status.h index e3273144e4..68efe3001a 100644 --- a/src/leveldb/include/leveldb/status.h +++ b/src/leveldb/include/leveldb/status.h @@ -103,6 +103,8 @@ class LEVELDB_EXPORT Status { inline Status::Status(const Status& rhs) { state_ = (rhs.state_ == nullptr) ? nullptr : CopyState(rhs.state_); } + +// NOLINTBEGIN(bugprone-unhandled-self-assignment) inline Status& Status::operator=(const Status& rhs) { // The following condition catches both aliasing (when this == &rhs), // and the common case where both rhs and *this are ok. @@ -112,6 +114,8 @@ inline Status& Status::operator=(const Status& rhs) { } return *this; } +// NOLINTEND(bugprone-unhandled-self-assignment) + inline Status& Status::operator=(Status&& rhs) noexcept { std::swap(state_, rhs.state_); return *this; diff --git a/src/logging.cpp b/src/logging.cpp index 578650f856..a9fea433be 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -13,6 +13,10 @@ #include <map> #include <optional> +using util::Join; +using util::RemovePrefix; +using util::ToString; + const char * const DEFAULT_DEBUGLOGFILE = "debug.log"; constexpr auto MAX_USER_SETABLE_SEVERITY_LEVEL{BCLog::Level::Info}; diff --git a/src/logging.h b/src/logging.h index cfef65221f..fe6b7051ba 100644 --- a/src/logging.h +++ b/src/logging.h @@ -189,7 +189,7 @@ namespace BCLog { /** Returns a string with the log categories in alphabetical order. */ std::string LogCategoriesString() const { - return Join(LogCategoriesList(), ", ", [&](const LogCategory& i) { return i.category; }); + return util::Join(LogCategoriesList(), ", ", [&](const LogCategory& i) { return i.category; }); }; //! Returns a string with all user-selectable log levels. diff --git a/src/mapport.cpp b/src/mapport.cpp index 80670230c7..1920297be6 100644 --- a/src/mapport.cpp +++ b/src/mapport.cpp @@ -161,8 +161,11 @@ static bool ProcessUpnp() struct UPNPUrls urls; struct IGDdatas data; int r; - +#if MINIUPNPC_API_VERSION <= 17 r = UPNP_GetValidIGD(devlist, &urls, &data, lanaddr, sizeof(lanaddr)); +#else + r = UPNP_GetValidIGD(devlist, &urls, &data, lanaddr, sizeof(lanaddr), nullptr, 0); +#endif if (r == 1) { if (fDiscover) { diff --git a/src/minisketch/configure.ac b/src/minisketch/configure.ac index cd52d7f412..65a47b45c2 100644 --- a/src/minisketch/configure.ac +++ b/src/minisketch/configure.ac @@ -102,6 +102,7 @@ case $host in esac AX_CHECK_COMPILE_FLAG([-Wall],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wall"],,[[$CXXFLAG_WERROR]]) +AX_CHECK_COMPILE_FLAG([-Wundef], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wundef"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-fvisibility=hidden],[CXXFLAGS="$CXXFLAGS -fvisibility=hidden"],[],[$CXXFLAG_WERROR]) if test "x$use_ccache" != "xno"; then diff --git a/src/minisketch/include/minisketch.h b/src/minisketch/include/minisketch.h index 24d6b4e1c0..b0571d2788 100644 --- a/src/minisketch/include/minisketch.h +++ b/src/minisketch/include/minisketch.h @@ -239,7 +239,7 @@ public: /** Make this Minisketch a clone of the specified one. */ Minisketch& operator=(const Minisketch& sketch) noexcept { - if (sketch.m_minisketch) { + if (this != &sketch && sketch.m_minisketch) { m_minisketch = std::unique_ptr<minisketch, Deleter>(minisketch_clone(sketch.m_minisketch.get())); } return *this; diff --git a/src/minisketch/src/false_positives.h b/src/minisketch/src/false_positives.h index 44ebb3e94c..9d0358997f 100644 --- a/src/minisketch/src/false_positives.h +++ b/src/minisketch/src/false_positives.h @@ -81,7 +81,8 @@ uint64_t BaseFPBits(uint32_t bits, uint32_t capacity) { size_t ComputeCapacity(uint32_t bits, size_t max_elements, uint32_t fpbits) { if (bits == 0) return 0; - uint64_t base_fpbits = BaseFPBits(bits, max_elements); + if (max_elements > 0xffffffff) return max_elements; + uint64_t base_fpbits = BaseFPBits(bits, static_cast<uint32_t>(max_elements)); // The fpbits provided by the base max_elements==capacity case are sufficient. if (base_fpbits >= fpbits) return max_elements; // Otherwise, increment capacity by ceil(fpbits / bits) beyond that. @@ -90,6 +91,7 @@ size_t ComputeCapacity(uint32_t bits, size_t max_elements, uint32_t fpbits) { size_t ComputeMaxElements(uint32_t bits, size_t capacity, uint32_t fpbits) { if (bits == 0) return 0; + if (capacity > 0xffffffff) return capacity; // Start with max_elements=capacity, and decrease max_elements until the corresponding capacity is capacity. size_t max_elements = capacity; while (true) { diff --git a/src/minisketch/src/int_utils.h b/src/minisketch/src/int_utils.h index 2b3d8cb402..a6b89cd63c 100644 --- a/src/minisketch/src/int_utils.h +++ b/src/minisketch/src/int_utils.h @@ -159,7 +159,7 @@ static inline int CountBits(I val, int max) { } if (!ret) return 0; return index + 1; -#elif HAVE_CLZ +#elif defined(HAVE_CLZ) (void)max; if (val == 0) return 0; if (std::numeric_limits<unsigned>::digits >= std::numeric_limits<I>::digits) { @@ -210,7 +210,7 @@ public: static constexpr inline int TopBits(I val) { static_assert(Count > 0, "BitsInt::TopBits needs Count > 0"); static_assert(Count <= BITS, "BitsInt::TopBits needs Offset <= BITS"); - return val >> (BITS - Count); + return static_cast<int>(val >> (BITS - Count)); } static inline constexpr I CondXorWith(I val, bool cond, I v) { diff --git a/src/minisketch/src/minisketch.cpp b/src/minisketch/src/minisketch.cpp index d003fdf755..2e45409243 100644 --- a/src/minisketch/src/minisketch.cpp +++ b/src/minisketch/src/minisketch.cpp @@ -468,7 +468,7 @@ size_t minisketch_merge(minisketch* sketch, const minisketch* other_sketch) { ssize_t minisketch_decode(const minisketch* sketch, size_t max_elements, uint64_t* output) { const Sketch* s = (const Sketch*)sketch; s->Check(); - return s->Decode(max_elements, output); + return s->Decode(static_cast<int>(max_elements), output); } void minisketch_set_seed(minisketch* sketch, uint64_t seed) { diff --git a/src/minisketch/src/sketch.h b/src/minisketch/src/sketch.h index 3e9bad793d..662b4e982f 100644 --- a/src/minisketch/src/sketch.h +++ b/src/minisketch/src/sketch.h @@ -29,7 +29,7 @@ public: virtual ~Sketch() {} virtual size_t Syndromes() const = 0; - virtual void Init(int syndromes) = 0; + virtual void Init(size_t syndromes) = 0; virtual void Add(uint64_t element) = 0; virtual void Serialize(unsigned char*) const = 0; virtual void Deserialize(const unsigned char*) = 0; diff --git a/src/minisketch/src/sketch_impl.h b/src/minisketch/src/sketch_impl.h index 4547b742f2..c357f0e823 100644 --- a/src/minisketch/src/sketch_impl.h +++ b/src/minisketch/src/sketch_impl.h @@ -92,7 +92,8 @@ template<typename F> void Sqr(std::vector<typename F::Elem>& poly, const F& field) { if (poly.size() == 0) return; poly.resize(poly.size() * 2 - 1); - for (int x = poly.size() - 1; x >= 0; --x) { + for (size_t i = 0; i < poly.size(); ++i) { + auto x = poly.size() - i - 1; poly[x] = (x & 1) ? 0 : field.Sqr(poly[x / 2]); } } @@ -217,7 +218,7 @@ bool RecFindRoots(std::vector<std::vector<typename F::Elem>>& stack, size_t pos, } if (fully_factorizable) { - // Every succesful iteration of this algorithm splits the input + // Every successful iteration of this algorithm splits the input // polynomial further into buckets, each corresponding to a subset // of 2^(BITS-depth) roots. If after depth splits the degree of // the polynomial is >= 2^(BITS-depth), something is wrong. @@ -297,7 +298,7 @@ std::vector<typename F::Elem> BerlekampMassey(const std::vector<typename F::Elem auto discrepancy = syndromes[n]; for (size_t i = 1; i < current.size(); ++i) discrepancy ^= table[n - i](current[i]); if (discrepancy != 0) { - int x = n + 1 - (current.size() - 1) - (prev.size() - 1); + int x = static_cast<int>(n + 1 - (current.size() - 1) - (prev.size() - 1)); if (!b_have_inv) { b_inv = field.Inv(b); b_have_inv = true; @@ -366,7 +367,7 @@ public: } size_t Syndromes() const override { return m_syndromes.size(); } - void Init(int count) override { m_syndromes.assign(count, 0); } + void Init(size_t count) override { m_syndromes.assign(count, 0); } void Add(uint64_t val) override { @@ -405,7 +406,7 @@ public: for (const auto& root : roots) { *(out++) = m_field.ToUint64(root); } - return roots.size(); + return static_cast<int>(roots.size()); } size_t Merge(const Sketch* other_sketch) override diff --git a/src/net_permissions.cpp b/src/net_permissions.cpp index b01b2f643d..8f0042c141 100644 --- a/src/net_permissions.cpp +++ b/src/net_permissions.cpp @@ -2,12 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/messages.h> #include <common/system.h> #include <net_permissions.h> #include <netbase.h> -#include <util/error.h> #include <util/translation.h> +using common::ResolveErrMsg; + const std::vector<std::string> NET_PERMISSIONS_DOC{ "bloomfilter (allow requesting BIP37 filtered blocks and transactions)", "noban (do not ban for misbehavior; implies download)", diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 461efc9253..40135a2cd3 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -25,6 +25,7 @@ #include <node/blockstorage.h> #include <node/timeoffsets.h> #include <node/txreconciliation.h> +#include <node/warnings.h> #include <policy/fees.h> #include <policy/policy.h> #include <policy/settings.h> @@ -482,7 +483,7 @@ class PeerManagerImpl final : public PeerManager public: PeerManagerImpl(CConnman& connman, AddrMan& addrman, BanMan* banman, ChainstateManager& chainman, - CTxMemPool& pool, Options opts); + CTxMemPool& pool, node::Warnings& warnings, Options opts); /** Overridden from CValidationInterface. */ void BlockConnected(ChainstateRole role, const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected) override @@ -777,7 +778,8 @@ private: /** Next time to check for stale tip */ std::chrono::seconds m_stale_tip_check_time GUARDED_BY(cs_main){0s}; - TimeOffsets m_outbound_time_offsets; + node::Warnings& m_warnings; + TimeOffsets m_outbound_time_offsets{m_warnings}; const Options m_opts; @@ -2012,14 +2014,14 @@ std::optional<std::string> PeerManagerImpl::FetchBlock(NodeId peer_id, const CBl std::unique_ptr<PeerManager> PeerManager::make(CConnman& connman, AddrMan& addrman, BanMan* banman, ChainstateManager& chainman, - CTxMemPool& pool, Options opts) + CTxMemPool& pool, node::Warnings& warnings, Options opts) { - return std::make_unique<PeerManagerImpl>(connman, addrman, banman, chainman, pool, opts); + return std::make_unique<PeerManagerImpl>(connman, addrman, banman, chainman, pool, warnings, opts); } PeerManagerImpl::PeerManagerImpl(CConnman& connman, AddrMan& addrman, BanMan* banman, ChainstateManager& chainman, - CTxMemPool& pool, Options opts) + CTxMemPool& pool, node::Warnings& warnings, Options opts) : m_rng{opts.deterministic_rng}, m_fee_filter_rounder{CFeeRate{DEFAULT_MIN_RELAY_TX_FEE}, m_rng}, m_chainparams(chainman.GetParams()), @@ -2028,6 +2030,7 @@ PeerManagerImpl::PeerManagerImpl(CConnman& connman, AddrMan& addrman, m_banman(banman), m_chainman(chainman), m_mempool(pool), + m_warnings{warnings}, m_opts{opts} { // While Erlay support is incomplete, it must be enabled explicitly via -txreconciliation. diff --git a/src/net_processing.h b/src/net_processing.h index 163f2e1385..bf9698ee02 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -16,6 +16,10 @@ class CChainParams; class CTxMemPool; class ChainstateManager; +namespace node { +class Warnings; +} // namespace node + /** Whether transaction reconciliation protocol should be enabled by default. */ static constexpr bool DEFAULT_TXRECONCILIATION_ENABLE{false}; /** Default for -maxorphantx, maximum number of orphan transactions kept in memory */ @@ -71,7 +75,7 @@ public: static std::unique_ptr<PeerManager> make(CConnman& connman, AddrMan& addrman, BanMan* banman, ChainstateManager& chainman, - CTxMemPool& pool, Options opts); + CTxMemPool& pool, node::Warnings& warnings, Options opts); virtual ~PeerManager() { } /** diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 74ab6dd8d8..0053464822 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -20,6 +20,9 @@ #include <iterator> #include <tuple> +using util::ContainsNoNUL; +using util::HasPrefix; + CNetAddr::BIP155Network CNetAddr::GetBIP155Network() const { switch (m_net) { diff --git a/src/netaddress.h b/src/netaddress.h index ea2d14336e..52fecada1c 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -448,7 +448,7 @@ private: // Recognize NET_INTERNAL embedded in IPv6, such addresses are not // gossiped but could be coming from addrman, when unserializing from // disk. - if (HasPrefix(m_addr, INTERNAL_IN_IPV6_PREFIX)) { + if (util::HasPrefix(m_addr, INTERNAL_IN_IPV6_PREFIX)) { m_net = NET_INTERNAL; memmove(m_addr.data(), m_addr.data() + INTERNAL_IN_IPV6_PREFIX.size(), ADDR_INTERNAL_SIZE); @@ -456,8 +456,8 @@ private: return; } - if (!HasPrefix(m_addr, IPV4_IN_IPV6_PREFIX) && - !HasPrefix(m_addr, TORV2_IN_IPV6_PREFIX)) { + if (!util::HasPrefix(m_addr, IPV4_IN_IPV6_PREFIX) && + !util::HasPrefix(m_addr, TORV2_IN_IPV6_PREFIX)) { return; } diff --git a/src/netbase.cpp b/src/netbase.cpp index e231766487..ff46061d3d 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -27,6 +27,8 @@ #include <sys/un.h> #endif +using util::ContainsNoNUL; + // Settings static GlobalMutex g_proxyinfo_mutex; static Proxy proxyInfo[NET_MAX] GUARDED_BY(g_proxyinfo_mutex); diff --git a/src/node/abort.cpp b/src/node/abort.cpp index b727608384..8a17c41fd2 100644 --- a/src/node/abort.cpp +++ b/src/node/abort.cpp @@ -6,19 +6,18 @@ #include <logging.h> #include <node/interface_ui.h> +#include <node/warnings.h> #include <util/signalinterrupt.h> #include <util/translation.h> -#include <warnings.h> #include <atomic> #include <cstdlib> -#include <string> namespace node { -void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const bilingual_str& message) +void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const bilingual_str& message, node::Warnings* warnings) { - SetMiscWarning(message); + if (warnings) warnings->Set(Warning::FATAL_INTERNAL_ERROR, message); InitError(_("A fatal internal error occurred, see debug.log for details: ") + message); exit_status.store(EXIT_FAILURE); if (shutdown && !(*shutdown)()) { diff --git a/src/node/abort.h b/src/node/abort.h index 1092279142..c881af4634 100644 --- a/src/node/abort.h +++ b/src/node/abort.h @@ -14,7 +14,8 @@ class SignalInterrupt; } // namespace util namespace node { -void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const bilingual_str& message); +class Warnings; +void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const bilingual_str& message, node::Warnings* warnings); } // namespace node #endif // BITCOIN_NODE_ABORT_H diff --git a/src/node/blockmanager_args.cpp b/src/node/blockmanager_args.cpp index dd8419a68a..fa76566652 100644 --- a/src/node/blockmanager_args.cpp +++ b/src/node/blockmanager_args.cpp @@ -33,8 +33,6 @@ util::Result<void> ApplyArgsManOptions(const ArgsManager& args, BlockManager::Op if (auto value{args.GetBoolArg("-fastprune")}) opts.fast_prune = *value; - if (auto value{args.GetBoolArg("-reindex")}) opts.reindex = *value; - return {}; } } // namespace node diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 4067ccee51..fb62e78138 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -551,7 +551,7 @@ bool BlockManager::LoadBlockIndexDB(const std::optional<uint256>& snapshot_block // Check whether we need to continue reindexing bool fReindexing = false; m_block_tree_db->ReadReindexing(fReindexing); - if (fReindexing) m_reindexing = true; + if (fReindexing) m_blockfiles_indexed = false; return true; } @@ -1182,7 +1182,7 @@ void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFile ImportingNow imp{chainman.m_blockman.m_importing}; // -reindex - if (chainman.m_blockman.m_reindexing) { + if (!chainman.m_blockman.m_blockfiles_indexed) { int nFile = 0; // Map of disk positions for blocks with unknown parent (only used for reindex); // parent hash -> child disk position, multiple children can have the same parent. @@ -1205,7 +1205,7 @@ void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFile nFile++; } WITH_LOCK(::cs_main, chainman.m_blockman.m_block_tree_db->WriteReindexing(false)); - chainman.m_blockman.m_reindexing = false; + chainman.m_blockman.m_blockfiles_indexed = true; LogPrintf("Reindexing finished\n"); // To avoid ending up in a situation without genesis block, re-try initializing (no-op if reindexing worked): chainman.ActiveChainstate().LoadGenesisBlock(); diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index a501067091..108a08a72b 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -267,18 +267,18 @@ public: explicit BlockManager(const util::SignalInterrupt& interrupt, Options opts) : m_prune_mode{opts.prune_target > 0}, m_opts{std::move(opts)}, - m_interrupt{interrupt}, - m_reindexing{m_opts.reindex} {}; + m_interrupt{interrupt} {} const util::SignalInterrupt& m_interrupt; std::atomic<bool> m_importing{false}; /** - * Tracks if a reindex is currently in progress. Set to true when a reindex - * is requested and false when reindexing completes. Its value is persisted - * in the BlockTreeDB across restarts. + * Whether all blockfiles have been added to the block tree database. + * Normally true, but set to false when a reindex is requested and the + * database is wiped. The value is persisted in the database across restarts + * and will be false until reindexing completes. */ - std::atomic_bool m_reindexing; + std::atomic_bool m_blockfiles_indexed{true}; BlockMap m_block_index GUARDED_BY(cs_main); @@ -359,7 +359,7 @@ public: [[nodiscard]] uint64_t GetPruneTarget() const { return m_opts.prune_target; } static constexpr auto PRUNE_TARGET_MANUAL{std::numeric_limits<uint64_t>::max()}; - [[nodiscard]] bool LoadingBlocks() const { return m_importing || m_reindexing; } + [[nodiscard]] bool LoadingBlocks() const { return m_importing || !m_blockfiles_indexed; } /** Calculate the amount of disk space the block & undo files currently use */ uint64_t CalculateCurrentUsage(); diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index d6eb14f513..d7e6176be1 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -45,11 +45,12 @@ static ChainstateLoadResult CompleteChainstateInitialization( .path = chainman.m_options.datadir / "blocks" / "index", .cache_bytes = static_cast<size_t>(cache_sizes.block_tree_db), .memory_only = options.block_tree_db_in_memory, - .wipe_data = options.reindex, + .wipe_data = options.wipe_block_tree_db, .options = chainman.m_options.block_tree_db}); - if (options.reindex) { + if (options.wipe_block_tree_db) { pblocktree->WriteReindexing(true); + chainman.m_blockman.m_blockfiles_indexed = false; //If we're reindexing in prune mode, wipe away unusable block files and all undo data files if (options.prune) { chainman.m_blockman.CleanupBlockRevFiles(); @@ -60,8 +61,7 @@ static ChainstateLoadResult CompleteChainstateInitialization( // LoadBlockIndex will load m_have_pruned if we've ever removed a // block file from disk. - // Note that it also sets m_reindexing based on the disk flag! - // From here on, m_reindexing and options.reindex values may be different! + // Note that it also sets m_blockfiles_indexed based on the disk flag! if (!chainman.LoadBlockIndex()) { if (chainman.m_interrupt) return {ChainstateLoadStatus::INTERRUPTED, {}}; return {ChainstateLoadStatus::FAILURE, _("Error loading block database")}; @@ -84,12 +84,12 @@ static ChainstateLoadResult CompleteChainstateInitialization( // If we're not mid-reindex (based on disk + args), add a genesis block on disk // (otherwise we use the one already on disk). // This is called again in ImportBlocks after the reindex completes. - if (!chainman.m_blockman.m_reindexing && !chainman.ActiveChainstate().LoadGenesisBlock()) { + if (chainman.m_blockman.m_blockfiles_indexed && !chainman.ActiveChainstate().LoadGenesisBlock()) { return {ChainstateLoadStatus::FAILURE, _("Error initializing block database")}; } auto is_coinsview_empty = [&](Chainstate* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { - return options.reindex || options.reindex_chainstate || chainstate->CoinsTip().GetBestBlock().IsNull(); + return options.wipe_chainstate_db || chainstate->CoinsTip().GetBestBlock().IsNull(); }; assert(chainman.m_total_coinstip_cache > 0); @@ -110,7 +110,7 @@ static ChainstateLoadResult CompleteChainstateInitialization( chainstate->InitCoinsDB( /*cache_size_bytes=*/chainman.m_total_coinsdb_cache * init_cache_fraction, /*in_memory=*/options.coins_db_in_memory, - /*should_wipe=*/options.reindex || options.reindex_chainstate); + /*should_wipe=*/options.wipe_chainstate_db); if (options.coins_error_cb) { chainstate->CoinsErrorCatcher().AddReadErrCallback(options.coins_error_cb); @@ -142,7 +142,7 @@ static ChainstateLoadResult CompleteChainstateInitialization( } } - if (!options.reindex) { + if (!options.wipe_block_tree_db) { auto chainstates{chainman.GetAll()}; if (std::any_of(chainstates.begin(), chainstates.end(), [](const Chainstate* cs) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return cs->NeedsRedownload(); })) { @@ -188,7 +188,7 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize // Load a chain created from a UTXO snapshot, if any exist. bool has_snapshot = chainman.DetectSnapshotChainstate(); - if (has_snapshot && (options.reindex || options.reindex_chainstate)) { + if (has_snapshot && options.wipe_chainstate_db) { LogPrintf("[snapshot] deleting snapshot chainstate due to reindexing\n"); if (!chainman.DeleteSnapshotChainstate()) { return {ChainstateLoadStatus::FAILURE_FATAL, Untranslated("Couldn't remove snapshot chainstate.")}; @@ -247,7 +247,7 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize ChainstateLoadResult VerifyLoadedChainstate(ChainstateManager& chainman, const ChainstateLoadOptions& options) { auto is_coinsview_empty = [&](Chainstate* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { - return options.reindex || options.reindex_chainstate || chainstate->CoinsTip().GetBestBlock().IsNull(); + return options.wipe_chainstate_db || chainstate->CoinsTip().GetBestBlock().IsNull(); }; LOCK(cs_main); diff --git a/src/node/chainstate.h b/src/node/chainstate.h index a6e9a0331b..bb0c4f2b87 100644 --- a/src/node/chainstate.h +++ b/src/node/chainstate.h @@ -22,8 +22,13 @@ struct ChainstateLoadOptions { CTxMemPool* mempool{nullptr}; bool block_tree_db_in_memory{false}; bool coins_db_in_memory{false}; - bool reindex{false}; - bool reindex_chainstate{false}; + // Whether to wipe the block tree database when loading it. If set, this + // will also set a reindexing flag so any existing block data files will be + // scanned and added to the database. + bool wipe_block_tree_db{false}; + // Whether to wipe the chainstate database when loading it. If set, this + // will cause the chainstate database to be rebuilt starting from genesis. + bool wipe_chainstate_db{false}; bool prune{false}; //! Setting require_full_verification to true will require all checks at //! check_level (below) to succeed for loading to succeed. Setting it to diff --git a/src/node/chainstatemanager_args.cpp b/src/node/chainstatemanager_args.cpp index 1cc126cb05..bc4a815a3e 100644 --- a/src/node/chainstatemanager_args.cpp +++ b/src/node/chainstatemanager_args.cpp @@ -24,7 +24,10 @@ namespace node { util::Result<void> ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts) { - if (auto value{args.GetBoolArg("-checkblockindex")}) opts.check_block_index = *value; + if (auto value{args.GetIntArg("-checkblockindex")}) { + // Interpret bare -checkblockindex argument as 1 instead of 0. + opts.check_block_index = args.GetArg("-checkblockindex")->empty() ? 1 : *value; + } if (auto value{args.GetBoolArg("-checkpoints")}) opts.checkpoints_enabled = *value; diff --git a/src/node/context.cpp b/src/node/context.cpp index e32d21b383..da05fde6ee 100644 --- a/src/node/context.cpp +++ b/src/node/context.cpp @@ -13,6 +13,7 @@ #include <net_processing.h> #include <netgroup.h> #include <node/kernel_notifications.h> +#include <node/warnings.h> #include <policy/fees.h> #include <scheduler.h> #include <txmempool.h> diff --git a/src/node/context.h b/src/node/context.h index a7d92989dd..77838ea99b 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -39,6 +39,7 @@ class SignalInterrupt; namespace node { class KernelNotifications; +class Warnings; //! NodeContext struct containing references to chain state and connection //! state. @@ -81,6 +82,8 @@ struct NodeContext { //! Issues calls about blocks and transactions std::unique_ptr<ValidationSignals> validation_signals; std::atomic<int> exit_status{EXIT_SUCCESS}; + //! Manages all the node warnings + std::unique_ptr<node::Warnings> warnings; //! Declare default constructor and destructor that are not inline, so code //! instantiating the NodeContext struct doesn't need to #include class diff --git a/src/node/interface_ui.cpp b/src/node/interface_ui.cpp index 9dd1e7d9cf..4f4d240d1b 100644 --- a/src/node/interface_ui.cpp +++ b/src/node/interface_ui.cpp @@ -10,6 +10,8 @@ #include <boost/signals2/optional_last_value.hpp> #include <boost/signals2/signal.hpp> +using util::MakeUnorderedList; + CClientUIInterface uiInterface; struct UISignals { diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 216f44ab9e..2b36f4ceae 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -31,6 +31,8 @@ #include <node/interface_ui.h> #include <node/mini_miner.h> #include <node/transaction.h> +#include <node/types.h> +#include <node/warnings.h> #include <policy/feerate.h> #include <policy/fees.h> #include <policy/policy.h> @@ -52,7 +54,6 @@ #include <util/translation.h> #include <validation.h> #include <validationinterface.h> -#include <warnings.h> #include <config/bitcoin-config.h> // IWYU pragma: keep @@ -70,6 +71,7 @@ using interfaces::Handler; using interfaces::MakeSignalHandler; using interfaces::Node; using interfaces::WalletLoader; +using util::Join; namespace node { // All members of the classes in this namespace are intentionally public, as the @@ -91,7 +93,7 @@ public: explicit NodeImpl(NodeContext& context) { setContext(&context); } void initLogging() override { InitLogging(args()); } void initParameterInteraction() override { InitParameterInteraction(args()); } - bilingual_str getWarnings() override { return Join(GetWarnings(), Untranslated("<hr />")); } + bilingual_str getWarnings() override { return Join(Assert(m_context->warnings)->GetMessages(), Untranslated("<hr />")); } int getExitStatus() override { return Assert(m_context)->exit_status.load(); } uint32_t getLogCategories() override { return LogInstance().GetCategoryMask(); } bool baseInitialize() override @@ -99,6 +101,7 @@ public: if (!AppInitBasicSetup(args(), Assert(context())->exit_status)) return false; if (!AppInitParameterInteraction(args())) return false; + m_context->warnings = std::make_unique<node::Warnings>(); m_context->kernel = std::make_unique<kernel::Context>(); m_context->ecc_context = std::make_unique<ECC_Context>(); if (!AppInitSanityChecks(*m_context->kernel)) return false; @@ -123,7 +126,7 @@ public: void startShutdown() override { if (!(*Assert(Assert(m_context)->shutdown))()) { - LogPrintf("Error: failed to send shutdown signal\n"); + LogError("Failed to send shutdown signal\n"); } // Stop RPC for clean shutdown if any of waitfor* commands is executed. if (args().GetBoolArg("-server", false)) { diff --git a/src/node/kernel_notifications.cpp b/src/node/kernel_notifications.cpp index e326d4a1f2..9894052a3a 100644 --- a/src/node/kernel_notifications.cpp +++ b/src/node/kernel_notifications.cpp @@ -10,23 +10,25 @@ #include <common/args.h> #include <common/system.h> #include <kernel/context.h> +#include <kernel/warning.h> #include <logging.h> #include <node/abort.h> #include <node/interface_ui.h> +#include <node/warnings.h> #include <util/check.h> #include <util/signalinterrupt.h> #include <util/strencodings.h> #include <util/string.h> #include <util/translation.h> -#include <warnings.h> #include <cstdint> #include <string> #include <thread> +using util::ReplaceAll; + static void AlertNotify(const std::string& strMessage) { - uiInterface.NotifyAlertChanged(); #if HAVE_SYSTEM std::string strCmd = gArgs.GetArg("-alertnotify", ""); if (strCmd.empty()) return; @@ -44,16 +46,6 @@ static void AlertNotify(const std::string& strMessage) #endif } -static void DoWarning(const bilingual_str& warning) -{ - static bool fWarned = false; - SetMiscWarning(warning); - if (!fWarned) { - AlertNotify(warning.original); - fWarned = true; - } -} - namespace node { kernel::InterruptResult KernelNotifications::blockTip(SynchronizationState state, CBlockIndex& index) @@ -61,7 +53,7 @@ kernel::InterruptResult KernelNotifications::blockTip(SynchronizationState state uiInterface.NotifyBlockTip(state, &index); if (m_stop_at_height && index.nHeight >= m_stop_at_height) { if (!m_shutdown()) { - LogPrintf("Error: failed to send shutdown signal after reaching stop height\n"); + LogError("Failed to send shutdown signal after reaching stop height\n"); } return kernel::Interrupted{}; } @@ -78,20 +70,27 @@ void KernelNotifications::progress(const bilingual_str& title, int progress_perc uiInterface.ShowProgress(title.translated, progress_percent, resume_possible); } -void KernelNotifications::warning(const bilingual_str& warning) +void KernelNotifications::warningSet(kernel::Warning id, const bilingual_str& message) +{ + if (m_warnings.Set(id, message)) { + AlertNotify(message.original); + } +} + +void KernelNotifications::warningUnset(kernel::Warning id) { - DoWarning(warning); + m_warnings.Unset(id); } void KernelNotifications::flushError(const bilingual_str& message) { - AbortNode(&m_shutdown, m_exit_status, message); + AbortNode(&m_shutdown, m_exit_status, message, &m_warnings); } void KernelNotifications::fatalError(const bilingual_str& message) { node::AbortNode(m_shutdown_on_fatal_error ? &m_shutdown : nullptr, - m_exit_status, message); + m_exit_status, message, &m_warnings); } void ReadNotificationArgs(const ArgsManager& args, KernelNotifications& notifications) diff --git a/src/node/kernel_notifications.h b/src/node/kernel_notifications.h index f4d97a0fff..e37f4d4e1e 100644 --- a/src/node/kernel_notifications.h +++ b/src/node/kernel_notifications.h @@ -15,18 +15,24 @@ class CBlockIndex; enum class SynchronizationState; struct bilingual_str; +namespace kernel { +enum class Warning; +} // namespace kernel + namespace util { class SignalInterrupt; } // namespace util namespace node { +class Warnings; static constexpr int DEFAULT_STOPATHEIGHT{0}; class KernelNotifications : public kernel::Notifications { public: - KernelNotifications(util::SignalInterrupt& shutdown, std::atomic<int>& exit_status) : m_shutdown(shutdown), m_exit_status{exit_status} {} + KernelNotifications(util::SignalInterrupt& shutdown, std::atomic<int>& exit_status, node::Warnings& warnings) + : m_shutdown(shutdown), m_exit_status{exit_status}, m_warnings{warnings} {} [[nodiscard]] kernel::InterruptResult blockTip(SynchronizationState state, CBlockIndex& index) override; @@ -34,7 +40,9 @@ public: void progress(const bilingual_str& title, int progress_percent, bool resume_possible) override; - void warning(const bilingual_str& warning) override; + void warningSet(kernel::Warning id, const bilingual_str& message) override; + + void warningUnset(kernel::Warning id) override; void flushError(const bilingual_str& message) override; @@ -47,6 +55,7 @@ public: private: util::SignalInterrupt& m_shutdown; std::atomic<int>& m_exit_status; + node::Warnings& m_warnings; }; void ReadNotificationArgs(const ArgsManager& args, KernelNotifications& notifications); diff --git a/src/node/mempool_args.cpp b/src/node/mempool_args.cpp index ac26600919..f329affb1d 100644 --- a/src/node/mempool_args.cpp +++ b/src/node/mempool_args.cpp @@ -8,19 +8,20 @@ #include <kernel/mempool_options.h> #include <common/args.h> +#include <common/messages.h> #include <consensus/amount.h> #include <kernel/chainparams.h> #include <logging.h> #include <policy/feerate.h> #include <policy/policy.h> #include <tinyformat.h> -#include <util/error.h> #include <util/moneystr.h> #include <util/translation.h> #include <chrono> #include <memory> +using common::AmountErrMsg; using kernel::MemPoolLimits; using kernel::MemPoolOptions; diff --git a/src/node/timeoffsets.cpp b/src/node/timeoffsets.cpp index 62f527be8a..002c00d245 100644 --- a/src/node/timeoffsets.cpp +++ b/src/node/timeoffsets.cpp @@ -3,13 +3,12 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <logging.h> -#include <node/interface_ui.h> #include <node/timeoffsets.h> +#include <node/warnings.h> #include <sync.h> #include <tinyformat.h> #include <util/time.h> #include <util/translation.h> -#include <warnings.h> #include <algorithm> #include <chrono> @@ -49,8 +48,7 @@ bool TimeOffsets::WarnIfOutOfSync() const // when median == std::numeric_limits<int64_t>::min(), calling std::chrono::abs is UB auto median{std::max(Median(), std::chrono::seconds(std::numeric_limits<int64_t>::min() + 1))}; if (std::chrono::abs(median) <= WARN_THRESHOLD) { - SetMedianTimeOffsetWarning(std::nullopt); - uiInterface.NotifyAlertChanged(); + m_warnings.Unset(node::Warning::CLOCK_OUT_OF_SYNC); return false; } @@ -63,7 +61,6 @@ bool TimeOffsets::WarnIfOutOfSync() const "RPC methods to get more info." ), Ticks<std::chrono::minutes>(WARN_THRESHOLD))}; LogWarning("%s\n", msg.original); - SetMedianTimeOffsetWarning(msg); - uiInterface.NotifyAlertChanged(); + m_warnings.Set(node::Warning::CLOCK_OUT_OF_SYNC, msg); return true; } diff --git a/src/node/timeoffsets.h b/src/node/timeoffsets.h index 2b12584e12..eba706ac1e 100644 --- a/src/node/timeoffsets.h +++ b/src/node/timeoffsets.h @@ -11,8 +11,16 @@ #include <cstddef> #include <deque> +namespace node { +class Warnings; +} // namespace node + class TimeOffsets { +public: + TimeOffsets(node::Warnings& warnings) : m_warnings{warnings} {} + +private: //! Maximum number of timeoffsets stored. static constexpr size_t MAX_SIZE{50}; //! Minimum difference between system and network time for a warning to be raised. @@ -23,6 +31,8 @@ class TimeOffsets * positive offset means our peer's clock is ahead of our local clock. */ std::deque<std::chrono::seconds> m_offsets GUARDED_BY(m_mutex){}; + node::Warnings& m_warnings; + public: /** Add a new time offset sample. */ void Add(std::chrono::seconds offset) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index b66a4f2f39..591dcd698d 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -9,6 +9,7 @@ #include <net_processing.h> #include <node/blockstorage.h> #include <node/context.h> +#include <node/types.h> #include <txmempool.h> #include <validation.h> #include <validationinterface.h> diff --git a/src/node/transaction.h b/src/node/transaction.h index 6782536ace..5f524f4e28 100644 --- a/src/node/transaction.h +++ b/src/node/transaction.h @@ -5,9 +5,9 @@ #ifndef BITCOIN_NODE_TRANSACTION_H #define BITCOIN_NODE_TRANSACTION_H +#include <common/messages.h> #include <policy/feerate.h> #include <primitives/transaction.h> -#include <util/error.h> class CBlockIndex; class CTxMemPool; diff --git a/src/node/types.h b/src/node/types.h new file mode 100644 index 0000000000..0461e85f43 --- /dev/null +++ b/src/node/types.h @@ -0,0 +1,29 @@ +// Copyright (c) 2010-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +//! @file node/types.h is a home for public enum and struct type definitions +//! that are used by internally by node code, but also used externally by wallet +//! or GUI code. +//! +//! This file is intended to define only simple types that do not have external +//! dependencies. More complicated types should be defined in dedicated header +//! files. + +#ifndef BITCOIN_NODE_TYPES_H +#define BITCOIN_NODE_TYPES_H + +namespace node { +enum class TransactionError { + OK, //!< No error + MISSING_INPUTS, + ALREADY_IN_CHAIN, + MEMPOOL_REJECTED, + MEMPOOL_ERROR, + MAX_FEE_EXCEEDED, + MAX_BURN_EXCEEDED, + INVALID_PACKAGE, +}; +} // namespace node + +#endif // BITCOIN_NODE_TYPES_H diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h index 256a4a601d..a7c4135787 100644 --- a/src/node/utxo_snapshot.h +++ b/src/node/utxo_snapshot.h @@ -13,6 +13,7 @@ #include <sync.h> #include <uint256.h> #include <util/chaintype.h> +#include <util/check.h> #include <util/fs.h> #include <cstdint> @@ -31,6 +32,7 @@ class SnapshotMetadata { const uint16_t m_version{1}; const std::set<uint16_t> m_supported_versions{1}; + const MessageStartChars m_network_magic; public: //! The hash of the block that reflects the tip of the chain for the //! UTXO set contained in this snapshot. @@ -42,11 +44,15 @@ public: //! during snapshot load to estimate progress of UTXO set reconstruction. uint64_t m_coins_count = 0; - SnapshotMetadata() { } SnapshotMetadata( + const MessageStartChars network_magic) : + m_network_magic(network_magic) { } + SnapshotMetadata( + const MessageStartChars network_magic, const uint256& base_blockhash, const int base_blockheight, uint64_t coins_count) : + m_network_magic(network_magic), m_base_blockhash(base_blockhash), m_base_blockheight(base_blockheight), m_coins_count(coins_count) { } @@ -55,7 +61,7 @@ public: inline void Serialize(Stream& s) const { s << SNAPSHOT_MAGIC_BYTES; s << m_version; - s << Params().MessageStart(); + s << m_network_magic; s << m_base_blockheight; s << m_base_blockhash; s << m_coins_count; @@ -80,11 +86,13 @@ public: // Read the network magic (pchMessageStart) MessageStartChars message; s >> message; - if (!std::equal(message.begin(), message.end(), Params().MessageStart().data())) { - auto metadata_network = GetNetworkForMagic(message); + if (!std::equal(message.begin(), message.end(), m_network_magic.data())) { + auto metadata_network{GetNetworkForMagic(message)}; if (metadata_network) { std::string network_string{ChainTypeToString(metadata_network.value())}; - throw std::ios_base::failure(strprintf("The network of the snapshot (%s) does not match the network of this node (%s).", network_string, Params().GetChainTypeString())); + auto node_network{GetNetworkForMagic(m_network_magic)}; + std::string node_network_string{ChainTypeToString(node_network.value())}; + throw std::ios_base::failure(strprintf("The network of the snapshot (%s) does not match the network of this node (%s).", network_string, node_network_string)); } else { throw std::ios_base::failure("This snapshot has been created for an unrecognized network. This could be a custom signet, a new testnet or possibly caused by data corruption."); } diff --git a/src/node/warnings.cpp b/src/node/warnings.cpp new file mode 100644 index 0000000000..b99c845900 --- /dev/null +++ b/src/node/warnings.cpp @@ -0,0 +1,68 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <config/bitcoin-config.h> // IWYU pragma: keep + +#include <node/warnings.h> + +#include <common/system.h> +#include <node/interface_ui.h> +#include <sync.h> +#include <univalue.h> +#include <util/translation.h> + +#include <utility> +#include <vector> + +namespace node { +Warnings::Warnings() +{ + // Pre-release build warning + if (!CLIENT_VERSION_IS_RELEASE) { + m_warnings.insert( + {Warning::PRE_RELEASE_TEST_BUILD, + _("This is a pre-release test build - use at your own risk - do not use for mining or merchant applications")}); + } +} +bool Warnings::Set(warning_type id, bilingual_str message) +{ + LOCK(m_mutex); + const auto& [_, inserted]{m_warnings.insert({id, std::move(message)})}; + if (inserted) uiInterface.NotifyAlertChanged(); + return inserted; +} + +bool Warnings::Unset(warning_type id) +{ + auto success{WITH_LOCK(m_mutex, return m_warnings.erase(id))}; + if (success) uiInterface.NotifyAlertChanged(); + return success; +} + +std::vector<bilingual_str> Warnings::GetMessages() const +{ + LOCK(m_mutex); + std::vector<bilingual_str> messages; + messages.reserve(m_warnings.size()); + for (const auto& [id, msg] : m_warnings) { + messages.push_back(msg); + } + return messages; +} + +UniValue GetWarningsForRpc(const Warnings& warnings, bool use_deprecated) +{ + if (use_deprecated) { + const auto all_messages{warnings.GetMessages()}; + return all_messages.empty() ? "" : all_messages.back().original; + } + + UniValue messages{UniValue::VARR}; + for (auto&& message : warnings.GetMessages()) { + messages.push_back(std::move(message.original)); + } + return messages; +} +} // namespace node diff --git a/src/node/warnings.h b/src/node/warnings.h new file mode 100644 index 0000000000..24aeb8a922 --- /dev/null +++ b/src/node/warnings.h @@ -0,0 +1,90 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_WARNINGS_H +#define BITCOIN_NODE_WARNINGS_H + +#include <sync.h> +#include <util/translation.h> + +#include <map> +#include <variant> +#include <vector> + +class UniValue; + +namespace kernel { +enum class Warning; +} // namespace kernel + +namespace node { +enum class Warning { + CLOCK_OUT_OF_SYNC, + PRE_RELEASE_TEST_BUILD, + FATAL_INTERNAL_ERROR, +}; + +/** + * @class Warnings + * @brief Manages warning messages within a node. + * + * The Warnings class provides mechanisms to set, unset, and retrieve + * warning messages. It updates the GUI when warnings are changed. + * + * This class is designed to be non-copyable to ensure warnings + * are managed centrally. + */ +class Warnings +{ + typedef std::variant<kernel::Warning, node::Warning> warning_type; + + mutable Mutex m_mutex; + std::map<warning_type, bilingual_str> m_warnings GUARDED_BY(m_mutex); + +public: + Warnings(); + //! A warnings instance should always be passed by reference, never copied. + Warnings(const Warnings&) = delete; + Warnings& operator=(const Warnings&) = delete; + /** + * @brief Set a warning message. If a warning with the specified + * `id` is already active, false is returned and the new + * warning is ignored. If `id` does not yet exist, the + * warning is set, the UI is updated, and true is returned. + * + * @param[in] id Unique identifier of the warning. + * @param[in] message Warning message to be shown. + * + * @returns true if the warning was indeed set (i.e. there is no + * active warning with this `id`), otherwise false. + */ + bool Set(warning_type id, bilingual_str message) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); + /** + * @brief Unset a warning message. If a warning with the specified + * `id` is active, it is unset, the UI is updated, and true + * is returned. Otherwise, no warning is unset and false is + * returned. + * + * @param[in] id Unique identifier of the warning. + * + * @returns true if the warning was indeed unset (i.e. there is an + * active warning with this `id`), otherwise false. + */ + bool Unset(warning_type id) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); + /** Return potential problems detected by the node, sorted by the + * warning_type id */ + std::vector<bilingual_str> GetMessages() const EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); +}; + +/** + * RPC helper function that wraps warnings.GetMessages(). + * + * Returns a UniValue::VSTR with the latest warning if use_deprecated is + * set to true, or a UniValue::VARR with all warnings otherwise. + */ +UniValue GetWarningsForRpc(const Warnings& warnings, bool use_deprecated); +} // namespace node + +#endif // BITCOIN_NODE_WARNINGS_H diff --git a/src/outputtype.cpp b/src/outputtype.cpp index c72d9deacb..8c2b76494b 100644 --- a/src/outputtype.cpp +++ b/src/outputtype.cpp @@ -81,11 +81,11 @@ std::vector<CTxDestination> GetAllDestinationsForKey(const CPubKey& key) } } -CTxDestination AddAndGetDestinationForScript(FillableSigningProvider& keystore, const CScript& script, OutputType type) +CTxDestination AddAndGetDestinationForScript(FlatSigningProvider& keystore, const CScript& script, OutputType type) { // Add script to keystore - keystore.AddCScript(script); - // Note that scripts over MAX_SCRIPT_ELEMENT_SIZE bytes are not yet supported. + keystore.scripts.emplace(CScriptID(script), script); + switch (type) { case OutputType::LEGACY: return ScriptHash(script); @@ -94,7 +94,7 @@ CTxDestination AddAndGetDestinationForScript(FillableSigningProvider& keystore, CTxDestination witdest = WitnessV0ScriptHash(script); CScript witprog = GetScriptForDestination(witdest); // Add the redeemscript, so that P2WSH and P2SH-P2WSH outputs are recognized as ours. - keystore.AddCScript(witprog); + keystore.scripts.emplace(CScriptID(witprog), witprog); if (type == OutputType::BECH32) { return witdest; } else { diff --git a/src/outputtype.h b/src/outputtype.h index a2d5942320..feef7991a6 100644 --- a/src/outputtype.h +++ b/src/outputtype.h @@ -46,7 +46,7 @@ std::vector<CTxDestination> GetAllDestinationsForKey(const CPubKey& key); * This function will automatically add the script (and any other * necessary scripts) to the keystore. */ -CTxDestination AddAndGetDestinationForScript(FillableSigningProvider& keystore, const CScript& script, OutputType); +CTxDestination AddAndGetDestinationForScript(FlatSigningProvider& keystore, const CScript& script, OutputType); /** Get the OutputType for a CTxDestination */ std::optional<OutputType> OutputTypeFromDestination(const CTxDestination& dest); diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index d8b4b907e4..e84a8428bf 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -93,7 +93,7 @@ bool IsStandard(const CScript& scriptPubKey, const std::optional<unsigned>& max_ bool IsStandardTx(const CTransaction& tx, const std::optional<unsigned>& max_datacarrier_bytes, bool permit_bare_multisig, const CFeeRate& dust_relay_fee, std::string& reason) { - if (tx.nVersion > TX_MAX_STANDARD_VERSION || tx.nVersion < 1) { + if (tx.version > TX_MAX_STANDARD_VERSION || tx.version < 1) { reason = "version"; return false; } diff --git a/src/policy/policy.h b/src/policy/policy.h index 6a7980c312..a82488a28c 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -131,7 +131,7 @@ bool IsStandard(const CScript& scriptPubKey, const std::optional<unsigned>& max_ // Changing the default transaction version requires a two step process: first // adapting relay policy by bumping TX_MAX_STANDARD_VERSION, and then later // allowing the new transaction version in the wallet/RPC. -static constexpr decltype(CTransaction::nVersion) TX_MAX_STANDARD_VERSION{2}; +static constexpr decltype(CTransaction::version) TX_MAX_STANDARD_VERSION{3}; /** * Check for standard transaction types diff --git a/src/policy/v3_policy.cpp b/src/policy/v3_policy.cpp index d44832fceb..6bd043b8e3 100644 --- a/src/policy/v3_policy.cpp +++ b/src/policy/v3_policy.cpp @@ -43,13 +43,13 @@ struct ParentInfo { const Txid& m_txid; /** Wtxid used for debug string */ const Wtxid& m_wtxid; - /** nVersion used to check inheritance of v3 and non-v3 */ - decltype(CTransaction::nVersion) m_version; + /** version used to check inheritance of v3 and non-v3 */ + decltype(CTransaction::version) m_version; /** If parent is in mempool, whether it has any descendants in mempool. */ bool m_has_mempool_descendant; ParentInfo() = delete; - ParentInfo(const Txid& txid, const Wtxid& wtxid, decltype(CTransaction::nVersion) version, bool has_mempool_descendant) : + ParentInfo(const Txid& txid, const Wtxid& wtxid, decltype(CTransaction::version) version, bool has_mempool_descendant) : m_txid{txid}, m_wtxid{wtxid}, m_version{version}, m_has_mempool_descendant{has_mempool_descendant} {} @@ -66,7 +66,7 @@ std::optional<std::string> PackageV3Checks(const CTransactionRef& ptx, int64_t v const auto in_package_parents{FindInPackageParents(package, ptx)}; // Now we have all ancestors, so we can start checking v3 rules. - if (ptx->nVersion == 3) { + if (ptx->version == TRUC_VERSION) { // SingleV3Checks should have checked this already. if (!Assume(vsize <= V3_MAX_VSIZE)) { return strprintf("v3 tx %s (wtxid=%s) is too big: %u > %u virtual bytes", @@ -91,23 +91,22 @@ std::optional<std::string> PackageV3Checks(const CTransactionRef& ptx, int64_t v const auto parent_info = [&] { if (mempool_ancestors.size() > 0) { auto& mempool_parent = *mempool_ancestors.begin(); - Assume(mempool_parent->GetCountWithDescendants() == 1); return ParentInfo{mempool_parent->GetTx().GetHash(), mempool_parent->GetTx().GetWitnessHash(), - mempool_parent->GetTx().nVersion, + mempool_parent->GetTx().version, /*has_mempool_descendant=*/mempool_parent->GetCountWithDescendants() > 1}; } else { auto& parent_index = in_package_parents.front(); auto& package_parent = package.at(parent_index); return ParentInfo{package_parent->GetHash(), package_parent->GetWitnessHash(), - package_parent->nVersion, + package_parent->version, /*has_mempool_descendant=*/false}; } }(); // If there is a parent, it must have the right version. - if (parent_info.m_version != 3) { + if (parent_info.m_version != TRUC_VERSION) { return strprintf("v3 tx %s (wtxid=%s) cannot spend from non-v3 tx %s (wtxid=%s)", ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString(), parent_info.m_txid.ToString(), parent_info.m_wtxid.ToString()); @@ -135,10 +134,7 @@ std::optional<std::string> PackageV3Checks(const CTransactionRef& ptx, int64_t v } } - // It shouldn't be possible to have any mempool siblings at this point. SingleV3Checks - // catches mempool siblings and sibling eviction is not extended to packages. Also, if the package consists of connected transactions, - // any tx having a mempool ancestor would mean the package exceeds ancestor limits. - if (!Assume(!parent_info.m_has_mempool_descendant)) { + if (parent_info.m_has_mempool_descendant) { return strprintf("tx %s (wtxid=%s) would exceed descendant count limit", parent_info.m_txid.ToString(), parent_info.m_wtxid.ToString()); } @@ -146,14 +142,14 @@ std::optional<std::string> PackageV3Checks(const CTransactionRef& ptx, int64_t v } else { // Non-v3 transactions cannot have v3 parents. for (auto it : mempool_ancestors) { - if (it->GetTx().nVersion == 3) { + if (it->GetTx().version == TRUC_VERSION) { return strprintf("non-v3 tx %s (wtxid=%s) cannot spend from v3 tx %s (wtxid=%s)", ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString(), it->GetSharedTx()->GetHash().ToString(), it->GetSharedTx()->GetWitnessHash().ToString()); } } for (const auto& index: in_package_parents) { - if (package.at(index)->nVersion == 3) { + if (package.at(index)->version == TRUC_VERSION) { return strprintf("non-v3 tx %s (wtxid=%s) cannot spend from v3 tx %s (wtxid=%s)", ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString(), @@ -172,12 +168,12 @@ std::optional<std::pair<std::string, CTransactionRef>> SingleV3Checks(const CTra { // Check v3 and non-v3 inheritance. for (const auto& entry : mempool_ancestors) { - if (ptx->nVersion != 3 && entry->GetTx().nVersion == 3) { + if (ptx->version != TRUC_VERSION && entry->GetTx().version == TRUC_VERSION) { return std::make_pair(strprintf("non-v3 tx %s (wtxid=%s) cannot spend from v3 tx %s (wtxid=%s)", ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString(), entry->GetSharedTx()->GetHash().ToString(), entry->GetSharedTx()->GetWitnessHash().ToString()), nullptr); - } else if (ptx->nVersion == 3 && entry->GetTx().nVersion != 3) { + } else if (ptx->version == TRUC_VERSION && entry->GetTx().version != TRUC_VERSION) { return std::make_pair(strprintf("v3 tx %s (wtxid=%s) cannot spend from non-v3 tx %s (wtxid=%s)", ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString(), entry->GetSharedTx()->GetHash().ToString(), entry->GetSharedTx()->GetWitnessHash().ToString()), @@ -189,8 +185,8 @@ std::optional<std::pair<std::string, CTransactionRef>> SingleV3Checks(const CTra static_assert(V3_ANCESTOR_LIMIT == 2); static_assert(V3_DESCENDANT_LIMIT == 2); - // The rest of the rules only apply to transactions with nVersion=3. - if (ptx->nVersion != 3) return std::nullopt; + // The rest of the rules only apply to transactions with version=3. + if (ptx->version != TRUC_VERSION) return std::nullopt; if (vsize > V3_MAX_VSIZE) { return std::make_pair(strprintf("v3 tx %s (wtxid=%s) is too big: %u > %u virtual bytes", diff --git a/src/policy/v3_policy.h b/src/policy/v3_policy.h index 25aff37a1b..90eaeda46f 100644 --- a/src/policy/v3_policy.h +++ b/src/policy/v3_policy.h @@ -15,8 +15,9 @@ #include <set> #include <string> -// This module enforces rules for transactions with nVersion=3 ("v3 transactions") which help make +// This module enforces rules for BIP 431 TRUC transactions (with version=3) which help make // RBF abilities more robust. +static constexpr decltype(CTransaction::version) TRUC_VERSION{3}; // v3 only allows 1 parent and 1 child when unconfirmed. /** Maximum number of transactions including an unconfirmed tx and its descendants. */ diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index b4a860dd9e..fab5c40765 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -6,12 +6,12 @@ #include <primitives/transaction.h> #include <consensus/amount.h> +#include <crypto/hex_base.h> #include <hash.h> #include <script/script.h> #include <serialize.h> #include <tinyformat.h> #include <uint256.h> -#include <util/strencodings.h> #include <util/transaction_identifier.h> #include <algorithm> @@ -63,8 +63,8 @@ std::string CTxOut::ToString() const return strprintf("CTxOut(nValue=%d.%08d, scriptPubKey=%s)", nValue / COIN, nValue % COIN, HexStr(scriptPubKey).substr(0, 30)); } -CMutableTransaction::CMutableTransaction() : nVersion(CTransaction::CURRENT_VERSION), nLockTime(0) {} -CMutableTransaction::CMutableTransaction(const CTransaction& tx) : vin(tx.vin), vout(tx.vout), nVersion(tx.nVersion), nLockTime(tx.nLockTime) {} +CMutableTransaction::CMutableTransaction() : version{CTransaction::CURRENT_VERSION}, nLockTime{0} {} +CMutableTransaction::CMutableTransaction(const CTransaction& tx) : vin(tx.vin), vout(tx.vout), version{tx.version}, nLockTime{tx.nLockTime} {} Txid CMutableTransaction::GetHash() const { @@ -92,8 +92,8 @@ Wtxid CTransaction::ComputeWitnessHash() const return Wtxid::FromUint256((HashWriter{} << TX_WITH_WITNESS(*this)).GetHash()); } -CTransaction::CTransaction(const CMutableTransaction& tx) : vin(tx.vin), vout(tx.vout), nVersion(tx.nVersion), nLockTime(tx.nLockTime), m_has_witness{ComputeHasWitness()}, hash{ComputeHash()}, m_witness_hash{ComputeWitnessHash()} {} -CTransaction::CTransaction(CMutableTransaction&& tx) : vin(std::move(tx.vin)), vout(std::move(tx.vout)), nVersion(tx.nVersion), nLockTime(tx.nLockTime), m_has_witness{ComputeHasWitness()}, hash{ComputeHash()}, m_witness_hash{ComputeWitnessHash()} {} +CTransaction::CTransaction(const CMutableTransaction& tx) : vin(tx.vin), vout(tx.vout), version{tx.version}, nLockTime{tx.nLockTime}, m_has_witness{ComputeHasWitness()}, hash{ComputeHash()}, m_witness_hash{ComputeWitnessHash()} {} +CTransaction::CTransaction(CMutableTransaction&& tx) : vin(std::move(tx.vin)), vout(std::move(tx.vout)), version{tx.version}, nLockTime{tx.nLockTime}, m_has_witness{ComputeHasWitness()}, hash{ComputeHash()}, m_witness_hash{ComputeWitnessHash()} {} CAmount CTransaction::GetValueOut() const { @@ -115,9 +115,9 @@ unsigned int CTransaction::GetTotalSize() const std::string CTransaction::ToString() const { std::string str; - str += strprintf("CTransaction(hash=%s, ver=%d, vin.size=%u, vout.size=%u, nLockTime=%u)\n", + str += strprintf("CTransaction(hash=%s, ver=%u, vin.size=%u, vout.size=%u, nLockTime=%u)\n", GetHash().ToString().substr(0,10), - nVersion, + version, vin.size(), vout.size(), nLockTime); diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 976542cfae..bf86562886 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -197,13 +197,13 @@ static constexpr TransactionSerParams TX_NO_WITNESS{.allow_witness = false}; /** * Basic transaction serialization format: - * - int32_t nVersion + * - uint32_t version * - std::vector<CTxIn> vin * - std::vector<CTxOut> vout * - uint32_t nLockTime * * Extended transaction serialization format: - * - int32_t nVersion + * - uint32_t version * - unsigned char dummy = 0x00 * - unsigned char flags (!= 0) * - std::vector<CTxIn> vin @@ -217,7 +217,7 @@ void UnserializeTransaction(TxType& tx, Stream& s, const TransactionSerParams& p { const bool fAllowWitness = params.allow_witness; - s >> tx.nVersion; + s >> tx.version; unsigned char flags = 0; tx.vin.clear(); tx.vout.clear(); @@ -257,7 +257,7 @@ void SerializeTransaction(const TxType& tx, Stream& s, const TransactionSerParam { const bool fAllowWitness = params.allow_witness; - s << tx.nVersion; + s << tx.version; unsigned char flags = 0; // Consistency check if (fAllowWitness) { @@ -296,7 +296,7 @@ class CTransaction { public: // Default transaction version. - static const int32_t CURRENT_VERSION=2; + static const uint32_t CURRENT_VERSION{2}; // The local variables are made const to prevent unintended modification // without updating the cached hash value. However, CTransaction is not @@ -305,7 +305,7 @@ public: // structure, including the hash. const std::vector<CTxIn> vin; const std::vector<CTxOut> vout; - const int32_t nVersion; + const uint32_t version; const uint32_t nLockTime; private: @@ -378,7 +378,7 @@ struct CMutableTransaction { std::vector<CTxIn> vin; std::vector<CTxOut> vout; - int32_t nVersion; + uint32_t version; uint32_t nLockTime; explicit CMutableTransaction(); diff --git a/src/psbt.cpp b/src/psbt.cpp index b2ee3ce7a5..19d855e4c7 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -4,12 +4,12 @@ #include <psbt.h> +#include <node/types.h> #include <policy/policy.h> #include <script/signingprovider.h> #include <util/check.h> #include <util/strencodings.h> - PartiallySignedTransaction::PartiallySignedTransaction(const CMutableTransaction& tx) : tx(tx) { inputs.resize(tx.vin.size()); @@ -508,17 +508,17 @@ bool FinalizeAndExtractPSBT(PartiallySignedTransaction& psbtx, CMutableTransacti return true; } -TransactionError CombinePSBTs(PartiallySignedTransaction& out, const std::vector<PartiallySignedTransaction>& psbtxs) +bool CombinePSBTs(PartiallySignedTransaction& out, const std::vector<PartiallySignedTransaction>& psbtxs) { out = psbtxs[0]; // Copy the first one // Merge for (auto it = std::next(psbtxs.begin()); it != psbtxs.end(); ++it) { if (!out.Merge(*it)) { - return TransactionError::PSBT_MISMATCH; + return false; } } - return TransactionError::OK; + return true; } std::string PSBTRoleName(PSBTRole role) { diff --git a/src/psbt.h b/src/psbt.h index 3f74083717..4607304046 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -17,6 +17,10 @@ #include <optional> +namespace node { +enum class TransactionError; +} // namespace node + // Magic bytes static constexpr uint8_t PSBT_MAGIC_BYTES[5] = {'p', 's', 'b', 't', 0xff}; @@ -237,7 +241,7 @@ struct PSBTInput if (final_script_sig.empty() && final_script_witness.IsNull()) { // Write any partial signatures - for (auto sig_pair : partial_sigs) { + for (const auto& sig_pair : partial_sigs) { SerializeToVector(s, CompactSizeWriter(PSBT_IN_PARTIAL_SIG), Span{sig_pair.second.first}); s << sig_pair.second.second; } @@ -1263,9 +1267,9 @@ bool FinalizeAndExtractPSBT(PartiallySignedTransaction& psbtx, CMutableTransacti * * @param[out] out the combined PSBT, if successful * @param[in] psbtxs the PSBTs to combine - * @return error (OK if we successfully combined the transactions, other error if they were not compatible) + * @return True if we successfully combined the transactions, false if they were not compatible */ -[[nodiscard]] TransactionError CombinePSBTs(PartiallySignedTransaction& out, const std::vector<PartiallySignedTransaction>& psbtxs); +[[nodiscard]] bool CombinePSBTs(PartiallySignedTransaction& out, const std::vector<PartiallySignedTransaction>& psbtxs); //! Decode a base64ed PSBT into a PartiallySignedTransaction [[nodiscard]] bool DecodeBase64PSBT(PartiallySignedTransaction& decoded_psbt, const std::string& base64_psbt, std::string& error); diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 44a858c16b..6c5725533b 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -85,6 +85,8 @@ Q_DECLARE_METATYPE(uint256) Q_DECLARE_METATYPE(wallet::AddressPurpose) #endif // ENABLE_WALLET +using util::MakeUnorderedList; + static void RegisterMetaTypes() { // Register meta types used for QMetaObject::invokeMethod and Qt::QueuedConnection diff --git a/src/qt/psbtoperationsdialog.cpp b/src/qt/psbtoperationsdialog.cpp index 353709c7f5..5a4b4442f3 100644 --- a/src/qt/psbtoperationsdialog.cpp +++ b/src/qt/psbtoperationsdialog.cpp @@ -4,10 +4,12 @@ #include <qt/psbtoperationsdialog.h> +#include <common/messages.h> #include <core_io.h> #include <interfaces/node.h> #include <key_io.h> #include <node/psbt.h> +#include <node/types.h> #include <policy/policy.h> #include <qt/bitcoinunits.h> #include <qt/forms/ui_psbtoperationsdialog.h> @@ -20,9 +22,11 @@ #include <iostream> #include <string> +using common::TransactionErrorString; using node::AnalyzePSBT; using node::DEFAULT_MAX_RAW_TX_FEE_RATE; using node::PSBTAnalysis; +using node::TransactionError; PSBTOperationsDialog::PSBTOperationsDialog( QWidget* parent, WalletModel* wallet_model, ClientModel* client_model) : QDialog(parent, GUIUtil::dialog_flags), @@ -55,10 +59,10 @@ void PSBTOperationsDialog::openWithPSBT(PartiallySignedTransaction psbtx) bool complete = FinalizePSBT(psbtx); // Make sure all existing signatures are fully combined before checking for completeness. if (m_wallet_model) { size_t n_could_sign; - TransactionError err = m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, &n_could_sign, m_transaction_data, complete); - if (err != TransactionError::OK) { + const auto err{m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, &n_could_sign, m_transaction_data, complete)}; + if (err) { showStatus(tr("Failed to load transaction: %1") - .arg(QString::fromStdString(TransactionErrorString(err).translated)), + .arg(QString::fromStdString(PSBTErrorString(*err).translated)), StatusLevel::ERR); return; } @@ -79,11 +83,11 @@ void PSBTOperationsDialog::signTransaction() WalletModel::UnlockContext ctx(m_wallet_model->requestUnlock()); - TransactionError err = m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/true, /*bip32derivs=*/true, &n_signed, m_transaction_data, complete); + const auto err{m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/true, /*bip32derivs=*/true, &n_signed, m_transaction_data, complete)}; - if (err != TransactionError::OK) { + if (err) { showStatus(tr("Failed to sign transaction: %1") - .arg(QString::fromStdString(TransactionErrorString(err).translated)), StatusLevel::ERR); + .arg(QString::fromStdString(PSBTErrorString(*err).translated)), StatusLevel::ERR); return; } @@ -247,9 +251,9 @@ size_t PSBTOperationsDialog::couldSignInputs(const PartiallySignedTransaction &p size_t n_signed; bool complete; - TransactionError err = m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/false, &n_signed, m_transaction_data, complete); + const auto err{m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/false, &n_signed, m_transaction_data, complete)}; - if (err != TransactionError::OK) { + if (err) { return 0; } return n_signed; diff --git a/src/qt/recentrequeststablemodel.cpp b/src/qt/recentrequeststablemodel.cpp index 52d4e45d49..a8d54cdc0c 100644 --- a/src/qt/recentrequeststablemodel.cpp +++ b/src/qt/recentrequeststablemodel.cpp @@ -20,6 +20,8 @@ #include <QLatin1Char> #include <QLatin1String> +using util::ToString; + RecentRequestsTableModel::RecentRequestsTableModel(WalletModel *parent) : QAbstractTableModel(parent), walletModel(parent) { diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 702ca44395..edf417a7cb 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -48,6 +48,8 @@ #include <chrono> +using util::Join; + const int CONSOLE_HISTORY = 50; const int INITIAL_TRAFFIC_GRAPH_MINS = 30; const QSize FONT_RANGE(4, 40); diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 0d8c0f7a63..03173ec80e 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -20,6 +20,7 @@ #include <interfaces/node.h> #include <key_io.h> #include <node/interface_ui.h> +#include <node/types.h> #include <policy/fees.h> #include <txmempool.h> #include <validation.h> @@ -37,6 +38,7 @@ #include <QSettings> #include <QTextDocument> +using common::PSBTError; using wallet::CCoinControl; using wallet::DEFAULT_PAY_TX_FEE; @@ -442,26 +444,26 @@ void SendCoinsDialog::presentPSBT(PartiallySignedTransaction& psbtx) } bool SendCoinsDialog::signWithExternalSigner(PartiallySignedTransaction& psbtx, CMutableTransaction& mtx, bool& complete) { - TransactionError err; + std::optional<PSBTError> err; try { err = model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/true, /*bip32derivs=*/true, /*n_signed=*/nullptr, psbtx, complete); } catch (const std::runtime_error& e) { QMessageBox::critical(nullptr, tr("Sign failed"), e.what()); return false; } - if (err == TransactionError::EXTERNAL_SIGNER_NOT_FOUND) { + if (err == PSBTError::EXTERNAL_SIGNER_NOT_FOUND) { //: "External signer" means using devices such as hardware wallets. const QString msg = tr("External signer not found"); QMessageBox::critical(nullptr, msg, msg); return false; } - if (err == TransactionError::EXTERNAL_SIGNER_FAILED) { + if (err == PSBTError::EXTERNAL_SIGNER_FAILED) { //: "External signer" means using devices such as hardware wallets. const QString msg = tr("External signer failure"); QMessageBox::critical(nullptr, msg, msg); return false; } - if (err != TransactionError::OK) { + if (err) { tfm::format(std::cerr, "Failed to sign PSBT"); processSendCoinsReturn(WalletModel::TransactionCreationFailed); return false; @@ -501,9 +503,9 @@ void SendCoinsDialog::sendButtonClicked([[maybe_unused]] bool checked) PartiallySignedTransaction psbtx(mtx); bool complete = false; // Fill without signing - TransactionError err = model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, /*n_signed=*/nullptr, psbtx, complete); + const auto err{model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, /*n_signed=*/nullptr, psbtx, complete)}; assert(!complete); - assert(err == TransactionError::OK); + assert(!err); // Copy PSBT to clipboard and offer to save presentPSBT(psbtx); @@ -517,9 +519,9 @@ void SendCoinsDialog::sendButtonClicked([[maybe_unused]] bool checked) bool complete = false; // Always fill without signing first. This prevents an external signer // from being called prematurely and is not expensive. - TransactionError err = model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, /*n_signed=*/nullptr, psbtx, complete); + const auto err{model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, /*n_signed=*/nullptr, psbtx, complete)}; assert(!complete); - assert(err == TransactionError::OK); + assert(!err); send_failure = !signWithExternalSigner(psbtx, mtx, complete); // Don't broadcast when user rejects it on the device or there's a failure: broadcast = complete && !send_failure; diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index 4392d76328..012186ee4d 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -10,9 +10,9 @@ #include <qt/platformstyle.h> #include <qt/walletmodel.h> +#include <common/signmessage.h> // For MessageSign(), MessageVerify() #include <config/bitcoin-config.h> // IWYU pragma: keep #include <key_io.h> -#include <util/message.h> // For MessageSign(), MessageVerify() #include <wallet/wallet.h> #include <vector> diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index c7fe62f4e9..34b47c90a3 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -30,6 +30,7 @@ #include <QTimer> #include <QWindow> +using util::Join; using wallet::WALLET_FLAG_BLANK_WALLET; using wallet::WALLET_FLAG_DESCRIPTORS; using wallet::WALLET_FLAG_DISABLE_PRIVATE_KEYS; diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 87ad98a4cc..f8ce068e12 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -19,6 +19,7 @@ #include <interfaces/node.h> #include <key_io.h> #include <node/interface_ui.h> +#include <node/types.h> #include <psbt.h> #include <util/translation.h> #include <wallet/coincontrol.h> @@ -534,8 +535,8 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) // "Create Unsigned" clicked PartiallySignedTransaction psbtx(mtx); bool complete = false; - const TransactionError err = wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, nullptr, psbtx, complete); - if (err != TransactionError::OK || complete) { + const auto err{wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, nullptr, psbtx, complete)}; + if (err || complete) { QMessageBox::critical(nullptr, tr("Fee bump error"), tr("Can't draft transaction.")); return false; } diff --git a/src/rest.cpp b/src/rest.cpp index 4e9d8fd2b1..d43018f5af 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -39,6 +39,7 @@ using node::GetTransaction; using node::NodeContext; +using util::SplitString; static const size_t MAX_GETUTXOS_OUTPOINTS = 15; //allow a max of 15 outpoints to be queried at once static constexpr unsigned int MAX_REST_HEADERS_RESULTS = 2000; diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index a6c959797a..e785678614 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -29,6 +29,7 @@ #include <node/context.h> #include <node/transaction.h> #include <node/utxo_snapshot.h> +#include <node/warnings.h> #include <primitives/transaction.h> #include <rpc/server.h> #include <rpc/server_util.h> @@ -61,6 +62,9 @@ using kernel::CoinStatsHashType; using node::BlockManager; using node::NodeContext; using node::SnapshotMetadata; +using util::Join; +using util::MakeUnorderedList; +using util::ToString; struct CUpdatedBlock { @@ -1305,7 +1309,8 @@ RPCHelpMan getblockchaininfo() } } - obj.pushKV("warnings", GetNodeWarnings(IsDeprecatedRPCEnabled("warnings"))); + NodeContext& node = EnsureAnyNodeContext(request.context); + obj.pushKV("warnings", node::GetWarningsForRpc(*CHECK_NONFATAL(node.warnings), IsDeprecatedRPCEnabled("warnings"))); return obj; }, }; @@ -2697,7 +2702,7 @@ UniValue CreateUTXOSnapshot( tip->nHeight, tip->GetBlockHash().ToString(), fs::PathToString(path), fs::PathToString(temppath))); - SnapshotMetadata metadata{tip->GetBlockHash(), tip->nHeight, maybe_stats->coins_count}; + SnapshotMetadata metadata{chainstate.m_chainman.GetParams().MessageStart(), tip->GetBlockHash(), tip->nHeight, maybe_stats->coins_count}; afile << metadata; @@ -2809,7 +2814,7 @@ static RPCHelpMan loadtxoutset() "Couldn't open file " + path.utf8string() + " for reading."); } - SnapshotMetadata metadata; + SnapshotMetadata metadata{chainman.GetParams().MessageStart()}; try { afile >> metadata; } catch (const std::ios_base::failure& e) { diff --git a/src/rpc/fees.cpp b/src/rpc/fees.cpp index a7cec96746..aefe78162b 100644 --- a/src/rpc/fees.cpp +++ b/src/rpc/fees.cpp @@ -3,6 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/messages.h> #include <core_io.h> #include <node/context.h> #include <policy/feerate.h> @@ -14,7 +15,6 @@ #include <rpc/util.h> #include <txmempool.h> #include <univalue.h> -#include <util/fees.h> #include <validationinterface.h> #include <algorithm> @@ -22,6 +22,9 @@ #include <cmath> #include <string> +using common::FeeModeFromString; +using common::FeeModes; +using common::InvalidEstimateModeErrorMessage; using node::NodeContext; static RPCHelpMan estimatesmartfee() diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index edd45d9731..fd11f6cfeb 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -11,6 +11,7 @@ #include <core_io.h> #include <kernel/mempool_entry.h> #include <node/mempool_persist_args.h> +#include <node/types.h> #include <policy/rbf.h> #include <policy/settings.h> #include <primitives/transaction.h> @@ -32,6 +33,8 @@ using node::DEFAULT_MAX_BURN_AMOUNT; using node::DEFAULT_MAX_RAW_TX_FEE_RATE; using node::MempoolPath; using node::NodeContext; +using node::TransactionError; +using util::ToString; static RPCHelpMan sendrawtransaction() { @@ -82,7 +85,7 @@ static RPCHelpMan sendrawtransaction() CTransactionRef tx(MakeTransactionRef(std::move(mtx))); - const CFeeRate max_raw_tx_fee_rate{ParseFeeRate(self.Arg<UniValue>(1))}; + const CFeeRate max_raw_tx_fee_rate{ParseFeeRate(self.Arg<UniValue>("maxfeerate"))}; int64_t virtual_size = GetVirtualTransactionSize(*tx); CAmount max_raw_tx_fee = max_raw_tx_fee_rate.GetFee(virtual_size); @@ -162,7 +165,7 @@ static RPCHelpMan testmempoolaccept() "Array must contain between 1 and " + ToString(MAX_PACKAGE_COUNT) + " transactions."); } - const CFeeRate max_raw_tx_fee_rate{ParseFeeRate(self.Arg<UniValue>(1))}; + const CFeeRate max_raw_tx_fee_rate{ParseFeeRate(self.Arg<UniValue>("maxfeerate"))}; std::vector<CTransactionRef> txns; txns.reserve(raw_transactions.size()); @@ -873,7 +876,7 @@ static RPCHelpMan submitpackage() } // Fee check needs to be run with chainstate and package context - const CFeeRate max_raw_tx_fee_rate = ParseFeeRate(self.Arg<UniValue>(1)); + const CFeeRate max_raw_tx_fee_rate{ParseFeeRate(self.Arg<UniValue>("maxfeerate"))}; std::optional<CFeeRate> client_maxfeerate{max_raw_tx_fee_rate}; // 0-value is special; it's mapped to no sanity check if (max_raw_tx_fee_rate == CFeeRate(0)) { diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 606fc62dbe..0f6853ef37 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -20,6 +20,7 @@ #include <net.h> #include <node/context.h> #include <node/miner.h> +#include <node/warnings.h> #include <pow.h> #include <rpc/blockchain.h> #include <rpc/mining.h> @@ -47,6 +48,7 @@ using node::CBlockTemplate; using node::NodeContext; using node::RegenerateCommitments; using node::UpdateTime; +using util::ToString; /** * Return average network hashes per second based on the last 'lookup' blocks, @@ -453,7 +455,7 @@ static RPCHelpMan getmininginfo() obj.pushKV("networkhashps", getnetworkhashps().HandleRequest(request)); obj.pushKV("pooledtx", (uint64_t)mempool.size()); obj.pushKV("chain", chainman.GetParams().GetChainTypeString()); - obj.pushKV("warnings", GetNodeWarnings(IsDeprecatedRPCEnabled("warnings"))); + obj.pushKV("warnings", node::GetWarningsForRpc(*CHECK_NONFATAL(node.warnings), IsDeprecatedRPCEnabled("warnings"))); return obj; }, }; @@ -485,7 +487,7 @@ static RPCHelpMan prioritisetransaction() LOCK(cs_main); uint256 hash(ParseHashV(request.params[0], "txid")); - const auto dummy{self.MaybeArg<double>(1)}; + const auto dummy{self.MaybeArg<double>("dummy")}; CAmount nAmount = request.params[2].getInt<int64_t>(); if (dummy && *dummy != 0) { diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 4fc93291f5..1119a3e668 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -16,6 +16,7 @@ #include <netbase.h> #include <node/context.h> #include <node/protocol_version.h> +#include <node/warnings.h> #include <policy/settings.h> #include <protocol.h> #include <rpc/blockchain.h> @@ -35,6 +36,8 @@ #include <univalue.h> using node::NodeContext; +using util::Join; +using util::TrimString; const std::vector<std::string> CONNECTION_TYPE_DOC{ "outbound-full-relay (default automatic connections)", @@ -401,7 +404,7 @@ static RPCHelpMan addconnection() } else { throw JSONRPCError(RPC_INVALID_PARAMETER, self.ToString()); } - bool use_v2transport = self.Arg<bool>(2); + bool use_v2transport{self.Arg<bool>("v2transport")}; NodeContext& node = EnsureAnyNodeContext(request.context); CConnman& connman = EnsureConnman(node); @@ -713,7 +716,7 @@ static RPCHelpMan getnetworkinfo() } } obj.pushKV("localaddresses", std::move(localAddresses)); - obj.pushKV("warnings", GetNodeWarnings(IsDeprecatedRPCEnabled("warnings"))); + obj.pushKV("warnings", node::GetWarningsForRpc(*CHECK_NONFATAL(node.warnings), IsDeprecatedRPCEnabled("warnings"))); return obj; }, }; diff --git a/src/rpc/output_script.cpp b/src/rpc/output_script.cpp index f771b31db0..65a9be2762 100644 --- a/src/rpc/output_script.cpp +++ b/src/rpc/output_script.cpp @@ -139,8 +139,7 @@ static RPCHelpMan createmultisig() output_type = parsed.value(); } - // Construct using pay-to-script-hash: - FillableSigningProvider keystore; + FlatSigningProvider keystore; CScript inner; const CTxDestination dest = AddAndGetMultisigDestination(required, pubkeys, output_type, keystore, inner); diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 7fa6652f9e..75b538061d 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -16,6 +16,7 @@ #include <node/context.h> #include <node/psbt.h> #include <node/transaction.h> +#include <node/types.h> #include <policy/packages.h> #include <policy/policy.h> #include <policy/rbf.h> @@ -785,7 +786,7 @@ static RPCHelpMan signrawtransactionwithkey() throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed. Make sure the tx has at least one input."); } - FillableSigningProvider keystore; + FlatSigningProvider keystore; const UniValue& keys = request.params[1].get_array(); for (unsigned int idx = 0; idx < keys.size(); ++idx) { UniValue k = keys[idx]; @@ -793,7 +794,11 @@ static RPCHelpMan signrawtransactionwithkey() if (!key.IsValid()) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key"); } - keystore.AddKey(key); + + CPubKey pubkey = key.GetPubKey(); + CKeyID key_id = pubkey.GetID(); + keystore.pubkeys.emplace(key_id, pubkey); + keystore.keys.emplace(key_id, key); } // Fetch previous transactions (inputs): @@ -1485,9 +1490,8 @@ static RPCHelpMan combinepsbt() } PartiallySignedTransaction merged_psbt; - const TransactionError error = CombinePSBTs(merged_psbt, psbtxs); - if (error != TransactionError::OK) { - throw JSONRPCTransactionError(error); + if (!CombinePSBTs(merged_psbt, psbtxs)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "PSBTs not compatible (different transactions)"); } DataStream ssTx{}; @@ -1744,8 +1748,8 @@ static RPCHelpMan joinpsbts() } psbtxs.push_back(psbtx); // Choose the highest version number - if (static_cast<uint32_t>(psbtx.tx->nVersion) > best_version) { - best_version = static_cast<uint32_t>(psbtx.tx->nVersion); + if (psbtx.tx->version > best_version) { + best_version = psbtx.tx->version; } // Choose the lowest lock time if (psbtx.tx->nLockTime < best_locktime) { @@ -1756,7 +1760,7 @@ static RPCHelpMan joinpsbts() // Create a blank psbt where everything will be added PartiallySignedTransaction merged_psbt; merged_psbt.tx = CMutableTransaction(); - merged_psbt.tx->nVersion = static_cast<int32_t>(best_version); + merged_psbt.tx->version = best_version; merged_psbt.tx->nLockTime = best_locktime; // Merge @@ -1791,7 +1795,7 @@ static RPCHelpMan joinpsbts() PartiallySignedTransaction shuffled_psbt; shuffled_psbt.tx = CMutableTransaction(); - shuffled_psbt.tx->nVersion = merged_psbt.tx->nVersion; + shuffled_psbt.tx->version = merged_psbt.tx->version; shuffled_psbt.tx->nLockTime = merged_psbt.tx->nLockTime; for (int i : input_indices) { shuffled_psbt.AddInput(merged_psbt.tx->vin[i], merged_psbt.inputs[i]); diff --git a/src/rpc/rawtransaction_util.cpp b/src/rpc/rawtransaction_util.cpp index a62f90215a..53f943bb9e 100644 --- a/src/rpc/rawtransaction_util.cpp +++ b/src/rpc/rawtransaction_util.cpp @@ -181,7 +181,7 @@ static void TxInErrorToJSON(const CTxIn& txin, UniValue& vErrorsRet, const std:: vErrorsRet.push_back(std::move(entry)); } -void ParsePrevouts(const UniValue& prevTxsUnival, FillableSigningProvider* keystore, std::map<COutPoint, Coin>& coins) +void ParsePrevouts(const UniValue& prevTxsUnival, FlatSigningProvider* keystore, std::map<COutPoint, Coin>& coins) { if (!prevTxsUnival.isNull()) { const UniValue& prevTxs = prevTxsUnival.get_array(); @@ -247,11 +247,11 @@ void ParsePrevouts(const UniValue& prevTxsUnival, FillableSigningProvider* keyst // work from witnessScript when possible std::vector<unsigned char> scriptData(!ws.isNull() ? ParseHexV(ws, "witnessScript") : ParseHexV(rs, "redeemScript")); CScript script(scriptData.begin(), scriptData.end()); - keystore->AddCScript(script); + keystore->scripts.emplace(CScriptID(script), script); // Automatically also add the P2WSH wrapped version of the script (to deal with P2SH-P2WSH). // This is done for redeemScript only for compatibility, it is encouraged to use the explicit witnessScript field instead. CScript witness_output_script{GetScriptForDestination(WitnessV0ScriptHash(script))}; - keystore->AddCScript(witness_output_script); + keystore->scripts.emplace(CScriptID(witness_output_script), witness_output_script); if (!ws.isNull() && !rs.isNull()) { // if both witnessScript and redeemScript are provided, diff --git a/src/rpc/rawtransaction_util.h b/src/rpc/rawtransaction_util.h index 964d0b095b..40d6bbba87 100644 --- a/src/rpc/rawtransaction_util.h +++ b/src/rpc/rawtransaction_util.h @@ -12,7 +12,7 @@ #include <optional> struct bilingual_str; -class FillableSigningProvider; +struct FlatSigningProvider; class UniValue; struct CMutableTransaction; class Coin; @@ -38,7 +38,7 @@ void SignTransactionResultToJSON(CMutableTransaction& mtx, bool complete, const * @param keystore A pointer to the temporary keystore if there is one * @param coins Map of unspent outputs - coins in mempool and current chain UTXO set, may be extended by previous txns outputs after call */ -void ParsePrevouts(const UniValue& prevTxsUnival, FillableSigningProvider* keystore, std::map<COutPoint, Coin>& coins); +void ParsePrevouts(const UniValue& prevTxsUnival, FlatSigningProvider* keystore, std::map<COutPoint, Coin>& coins); /** Normalize univalue-represented inputs and add them to the transaction */ void AddInputs(CMutableTransaction& rawTx, const UniValue& inputs_in, bool rbf); diff --git a/src/rpc/request.cpp b/src/rpc/request.cpp index d35782189e..87b9f18b33 100644 --- a/src/rpc/request.cpp +++ b/src/rpc/request.cpp @@ -45,6 +45,7 @@ UniValue JSONRPCRequestObj(const std::string& strMethod, const UniValue& params, request.pushKV("method", strMethod); request.pushKV("params", params); request.pushKV("id", id); + request.pushKV("jsonrpc", "2.0"); return request; } diff --git a/src/rpc/request.h b/src/rpc/request.h index e47f90af86..9968426636 100644 --- a/src/rpc/request.h +++ b/src/rpc/request.h @@ -17,6 +17,7 @@ enum class JSONRPCVersion { V2 }; +/** JSON-RPC 2.0 request, only used in bitcoin-cli **/ UniValue JSONRPCRequestObj(const std::string& strMethod, const UniValue& params, const UniValue& id); UniValue JSONRPCReplyObj(UniValue result, UniValue error, std::optional<UniValue> id, JSONRPCVersion jsonrpc_version); UniValue JSONRPCError(int code, const std::string& message); diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 838068bc19..19063fa5be 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -27,6 +27,8 @@ #include <mutex> #include <unordered_map> +using util::SplitString; + static GlobalMutex g_rpc_warmup_mutex; static std::atomic<bool> g_rpc_running{false}; static bool fRPCInWarmup GUARDED_BY(g_rpc_warmup_mutex) = true; diff --git a/src/rpc/signmessage.cpp b/src/rpc/signmessage.cpp index 9f3c24efcf..83462738c5 100644 --- a/src/rpc/signmessage.cpp +++ b/src/rpc/signmessage.cpp @@ -3,6 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/signmessage.h> #include <key.h> #include <key_io.h> #include <rpc/protocol.h> @@ -10,7 +11,6 @@ #include <rpc/server.h> #include <rpc/util.h> #include <univalue.h> -#include <util/message.h> #include <string> diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index ed32af7646..4df4466c49 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -5,14 +5,17 @@ #include <config/bitcoin-config.h> // IWYU pragma: keep #include <clientversion.h> -#include <core_io.h> #include <common/args.h> +#include <common/messages.h> +#include <common/types.h> #include <consensus/amount.h> -#include <script/interpreter.h> +#include <core_io.h> #include <key_io.h> +#include <node/types.h> #include <outputtype.h> #include <rpc/util.h> #include <script/descriptor.h> +#include <script/interpreter.h> #include <script/signingprovider.h> #include <script/solver.h> #include <tinyformat.h> @@ -22,7 +25,6 @@ #include <util/strencodings.h> #include <util/string.h> #include <util/translation.h> -#include <warnings.h> #include <algorithm> #include <iterator> @@ -30,6 +32,14 @@ #include <tuple> #include <utility> +using common::PSBTError; +using common::PSBTErrorString; +using common::TransactionErrorString; +using node::TransactionError; +using util::Join; +using util::SplitString; +using util::TrimString; + const std::string UNIX_EPOCH_TIME = "UNIX epoch time"; const std::string EXAMPLE_ADDRESS[2] = {"bc1q09vm5lfy0j5reeulh4x5752q25uqqvz34hufdl", "bc1q02ad21edsxd23d32dfgqqsz4vv4nmtfzuklhy3"}; @@ -176,7 +186,7 @@ std::string HelpExampleCliNamed(const std::string& methodname, const RPCArgList& std::string HelpExampleRpc(const std::string& methodname, const std::string& args) { return "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", " - "\"method\": \"" + methodname + "\", \"params\": [" + args + "]}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"; + "\"method\": \"" + methodname + "\", \"params\": [" + args + "]}' -H 'content-type: application/json' http://127.0.0.1:8332/\n"; } std::string HelpExampleRpcNamed(const std::string& methodname, const RPCArgList& args) @@ -187,7 +197,7 @@ std::string HelpExampleRpcNamed(const std::string& methodname, const RPCArgList& } return "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", " - "\"method\": \"" + methodname + "\", \"params\": " + params.write() + "}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"; + "\"method\": \"" + methodname + "\", \"params\": " + params.write() + "}' -H 'content-type: application/json' http://127.0.0.1:8332/\n"; } // Converts a hex string to a public key if possible @@ -228,7 +238,7 @@ CPubKey AddrToPubKey(const FillableSigningProvider& keystore, const std::string& } // Creates a multisig address from a given list of public keys, number of signatures required, and the address type -CTxDestination AddAndGetMultisigDestination(const int required, const std::vector<CPubKey>& pubkeys, OutputType type, FillableSigningProvider& keystore, CScript& script_out) +CTxDestination AddAndGetMultisigDestination(const int required, const std::vector<CPubKey>& pubkeys, OutputType type, FlatSigningProvider& keystore, CScript& script_out) { // Gather public keys if (required < 1) { @@ -364,6 +374,18 @@ unsigned int ParseConfirmTarget(const UniValue& value, unsigned int max_target) return unsigned_target; } +RPCErrorCode RPCErrorFromPSBTError(PSBTError err) +{ + switch (err) { + case PSBTError::UNSUPPORTED: + return RPC_INVALID_PARAMETER; + case PSBTError::SIGHASH_MISMATCH: + return RPC_DESERIALIZATION_ERROR; + default: break; + } + return RPC_TRANSACTION_ERROR; +} + RPCErrorCode RPCErrorFromTransactionError(TransactionError terr) { switch (terr) { @@ -371,18 +393,16 @@ RPCErrorCode RPCErrorFromTransactionError(TransactionError terr) return RPC_TRANSACTION_REJECTED; case TransactionError::ALREADY_IN_CHAIN: return RPC_TRANSACTION_ALREADY_IN_CHAIN; - case TransactionError::P2P_DISABLED: - return RPC_CLIENT_P2P_DISABLED; - case TransactionError::INVALID_PSBT: - case TransactionError::PSBT_MISMATCH: - return RPC_INVALID_PARAMETER; - case TransactionError::SIGHASH_MISMATCH: - return RPC_DESERIALIZATION_ERROR; default: break; } return RPC_TRANSACTION_ERROR; } +UniValue JSONRPCPSBTError(PSBTError err) +{ + return JSONRPCError(RPCErrorFromPSBTError(err), PSBTErrorString(err).original); +} + UniValue JSONRPCTransactionError(TransactionError terr, const std::string& err_string) { if (err_string.length() > 0) { @@ -677,7 +697,7 @@ static const UniValue* DetailMaybeArg(CheckFn* check, const std::vector<RPCArg>& static void CheckRequiredOrDefault(const RPCArg& param) { - // Must use `Arg<Type>(i)` to get the argument or its default value. + // Must use `Arg<Type>(key)` to get the argument or its default value. const bool required{ std::holds_alternative<RPCArg::Optional>(param.m_fallback) && RPCArg::Optional::NO == std::get<RPCArg::Optional>(param.m_fallback), }; @@ -778,7 +798,7 @@ std::string RPCHelpMan::ToString() const if (arg.m_opts.hidden) break; // Any arg that follows is also hidden // Push named argument name and description - sections.m_sections.emplace_back(::ToString(i + 1) + ". " + arg.GetFirstName(), arg.ToDescriptionString(/*is_named_arg=*/true)); + sections.m_sections.emplace_back(util::ToString(i + 1) + ". " + arg.GetFirstName(), arg.ToDescriptionString(/*is_named_arg=*/true)); sections.m_max_pad = std::max(sections.m_max_pad, sections.m_sections.back().m_left.size()); // Recursively push nested args @@ -1361,17 +1381,3 @@ void PushWarnings(const std::vector<bilingual_str>& warnings, UniValue& obj) if (warnings.empty()) return; obj.pushKV("warnings", BilingualStringsToUniValue(warnings)); } - -UniValue GetNodeWarnings(bool use_deprecated) -{ - if (use_deprecated) { - const auto all_warnings{GetWarnings()}; - return all_warnings.empty() ? "" : all_warnings.back().original; - } - - UniValue warnings{UniValue::VARR}; - for (auto&& warning : GetWarnings()) { - warnings.push_back(std::move(warning.original)); - } - return warnings; -} diff --git a/src/rpc/util.h b/src/rpc/util.h index 0e4dcc27b5..23024376e0 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -34,9 +34,14 @@ class JSONRPCRequest; enum ServiceFlags : uint64_t; enum class OutputType; -enum class TransactionError; struct FlatSigningProvider; struct bilingual_str; +namespace common { +enum class PSBTError; +} // namespace common +namespace node { +enum class TransactionError; +} // namespace node static constexpr bool DEFAULT_RPC_DOC_CHECK{ #ifdef RPC_DOC_CHECK @@ -117,7 +122,7 @@ std::string HelpExampleRpcNamed(const std::string& methodname, const RPCArgList& CPubKey HexToPubKey(const std::string& hex_in); CPubKey AddrToPubKey(const FillableSigningProvider& keystore, const std::string& addr_in); -CTxDestination AddAndGetMultisigDestination(const int required, const std::vector<CPubKey>& pubkeys, OutputType type, FillableSigningProvider& keystore, CScript& script_out); +CTxDestination AddAndGetMultisigDestination(const int required, const std::vector<CPubKey>& pubkeys, OutputType type, FlatSigningProvider& keystore, CScript& script_out); UniValue DescribeAddress(const CTxDestination& dest); @@ -127,8 +132,9 @@ int ParseSighashString(const UniValue& sighash); //! Parse a confirm target option and raise an RPC error if it is invalid. unsigned int ParseConfirmTarget(const UniValue& value, unsigned int max_target); -RPCErrorCode RPCErrorFromTransactionError(TransactionError terr); -UniValue JSONRPCTransactionError(TransactionError terr, const std::string& err_string = ""); +RPCErrorCode RPCErrorFromTransactionError(node::TransactionError terr); +UniValue JSONRPCPSBTError(common::PSBTError err); +UniValue JSONRPCTransactionError(node::TransactionError terr, const std::string& err_string = ""); //! Parse a JSON range specified as int64, or [int64, int64] std::pair<int64_t, int64_t> ParseDescriptorRange(const UniValue& value); @@ -414,19 +420,16 @@ public: * argument isNull() and parses (from JSON) and returns the user-passed argument, * or the default value derived from the RPCArg documentation. * - * There are two overloads of this function: - * - Use Arg<Type>(size_t i) to get the argument (or the default value) by index. - * - Use Arg<Type>(const std::string& key) to get the argument (or the default value) by key. - * - * The Type passed to this helper must match the corresponding RPCArg::Type. + * The instantiation of this helper for type R must match the corresponding RPCArg::Type. * - * @return The value of the RPC argument (or the default value) cast to type Type. + * @return The value of the RPC argument (or the default value) cast to type R. * * @see MaybeArg for handling optional arguments without default values. */ template <typename R> - auto Arg(size_t i) const + auto Arg(std::string_view key) const { + auto i{GetParamIndex(key)}; // Return argument (required or with default value). if constexpr (std::is_integral_v<R> || std::is_floating_point_v<R>) { // Return numbers by value. @@ -436,11 +439,6 @@ public: return ArgValue<const R&>(i); } } - template<typename R> - auto Arg(std::string_view key) const - { - return Arg<R>(GetParamIndex(key)); - } /** * @brief Helper to get an optional request argument. * @@ -452,21 +450,18 @@ public: * argument isNull() and parses (from JSON) and returns the user-passed argument, * or a falsy value if no argument was passed. * - * There are two overloads of this function: - * - Use MaybeArg<Type>(size_t i) to get the optional argument by index. - * - Use MaybeArg<Type>(const std::string& key) to get the optional argument by key. + * The instantiation of this helper for type R must match the corresponding RPCArg::Type. * - * The Type passed to this helper must match the corresponding RPCArg::Type. + * @return For integral and floating-point types, a std::optional<R> is returned. + * For other types, a R* pointer to the argument is returned. If the + * argument is not provided, std::nullopt or a null pointer is returned. * - * @return For integral and floating-point types, a std::optional<Type> is returned. - * For other types, a Type* pointer to the argument is returned. If the - * argument is not provided, std::nullopt or a null pointer is returned. - * * @see Arg for handling arguments that are required or have a default value. */ template <typename R> - auto MaybeArg(size_t i) const + auto MaybeArg(std::string_view key) const { + auto i{GetParamIndex(key)}; // Return optional argument (without default). if constexpr (std::is_integral_v<R> || std::is_floating_point_v<R>) { // Return numbers by value, wrapped in optional. @@ -476,11 +471,6 @@ public: return ArgValue<const R*>(i); } } - template<typename R> - auto MaybeArg(std::string_view key) const - { - return MaybeArg<R>(GetParamIndex(key)); - } std::string ToString() const; /** Return the named args that need to be converted from string to another JSON type */ UniValue GetArgMap() const; @@ -513,6 +503,4 @@ private: void PushWarnings(const UniValue& warnings, UniValue& obj); void PushWarnings(const std::vector<bilingual_str>& warnings, UniValue& obj); -UniValue GetNodeWarnings(bool use_deprecated); - #endif // BITCOIN_RPC_UTIL_H diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index a11d4dcbd5..0987db194c 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -8,6 +8,7 @@ #include <key_io.h> #include <pubkey.h> #include <script/miniscript.h> +#include <script/parsing.h> #include <script/script.h> #include <script/signingprovider.h> #include <script/solver.h> @@ -17,7 +18,6 @@ #include <span.h> #include <util/bip32.h> #include <util/check.h> -#include <util/spanparsing.h> #include <util/strencodings.h> #include <util/vector.h> @@ -27,6 +27,8 @@ #include <string> #include <vector> +using util::Split; + namespace { //////////////////////////////////////////////////////////////////////////// @@ -1350,8 +1352,6 @@ enum class ParseScriptContext { /** Parse a public key that excludes origin information. */ std::unique_ptr<PubkeyProvider> ParsePubkeyInner(uint32_t key_exp_index, const Span<const char>& sp, ParseScriptContext ctx, FlatSigningProvider& out, bool& apostrophe, std::string& error) { - using namespace spanparsing; - bool permit_uncompressed = ctx == ParseScriptContext::TOP || ctx == ParseScriptContext::P2SH; auto split = Split(sp, '/'); std::string str(split[0].begin(), split[0].end()); @@ -1424,8 +1424,6 @@ std::unique_ptr<PubkeyProvider> ParsePubkeyInner(uint32_t key_exp_index, const S /** Parse a public key including origin information (if enabled). */ std::unique_ptr<PubkeyProvider> ParsePubkey(uint32_t key_exp_index, const Span<const char>& sp, ParseScriptContext ctx, FlatSigningProvider& out, std::string& error) { - using namespace spanparsing; - auto origin_split = Split(sp, ']'); if (origin_split.size() > 2) { error = "Multiple ']' characters found for a single pubkey"; @@ -1589,7 +1587,7 @@ struct KeyParser { // NOLINTNEXTLINE(misc-no-recursion) std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const char>& sp, ParseScriptContext ctx, FlatSigningProvider& out, std::string& error) { - using namespace spanparsing; + using namespace script; auto expr = Expr(sp); if (Func("pk", expr)) { @@ -2038,8 +2036,6 @@ std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptCo /** Check a descriptor checksum, and update desc to be the checksum-less part. */ bool CheckChecksum(Span<const char>& sp, bool require_checksum, std::string& error, std::string* out_checksum = nullptr) { - using namespace spanparsing; - auto check_split = Split(sp, '#'); if (check_split.size() > 2) { error = "Multiple '#' symbols"; diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index c969ce45f1..20a9830d0e 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1321,8 +1321,8 @@ public: /** Serialize txTo */ template<typename S> void Serialize(S &s) const { - // Serialize nVersion - ::Serialize(s, txTo.nVersion); + // Serialize version + ::Serialize(s, txTo.version); // Serialize vin unsigned int nInputs = fAnyoneCanPay ? 1 : txTo.vin.size(); ::WriteCompactSize(s, nInputs); @@ -1512,7 +1512,7 @@ bool SignatureHashSchnorr(uint256& hash_out, ScriptExecutionData& execdata, cons ss << hash_type; // Transaction level data - ss << tx_to.nVersion; + ss << tx_to.version; ss << tx_to.nLockTime; if (input_type != SIGHASH_ANYONECANPAY) { ss << cache.m_prevouts_single_hash; @@ -1594,7 +1594,7 @@ uint256 SignatureHash(const CScript& scriptCode, const T& txTo, unsigned int nIn HashWriter ss{}; // Version - ss << txTo.nVersion; + ss << txTo.version; // Input prevouts/nSequence (none/all, depending on flags) ss << hashPrevouts; ss << hashSequence; @@ -1743,7 +1743,7 @@ bool GenericTransactionSignatureChecker<T>::CheckSequence(const CScriptNum& nSeq // Fail if the transaction's version number is not set high // enough to trigger BIP 68 rules. - if (static_cast<uint32_t>(txTo->nVersion) < 2) + if (txTo->version < 2) return false; // Sequence numbers with their most significant bit set are not diff --git a/src/script/miniscript.h b/src/script/miniscript.h index 4880f32410..a269709e72 100644 --- a/src/script/miniscript.h +++ b/src/script/miniscript.h @@ -18,10 +18,10 @@ #include <policy/policy.h> #include <primitives/transaction.h> +#include <script/parsing.h> #include <script/script.h> #include <span.h> #include <util/check.h> -#include <util/spanparsing.h> #include <util/strencodings.h> #include <util/string.h> #include <util/vector.h> @@ -251,7 +251,7 @@ namespace internal { //! The maximum size of a witness item for a Miniscript under Tapscript context. (A BIP340 signature with a sighash type byte.) static constexpr uint32_t MAX_TAPMINISCRIPT_STACK_ELEM_SIZE{65}; -//! nVersion + nLockTime +//! version + nLockTime constexpr uint32_t TX_OVERHEAD{4 + 4}; //! prevout + nSequence + scriptSig constexpr uint32_t TXIN_BYTES_NO_WITNESS{36 + 4 + 1}; @@ -863,8 +863,8 @@ public: if (!key_str) return {}; return std::move(ret) + "pk_h(" + std::move(*key_str) + ")"; } - case Fragment::AFTER: return std::move(ret) + "after(" + ::ToString(node.k) + ")"; - case Fragment::OLDER: return std::move(ret) + "older(" + ::ToString(node.k) + ")"; + case Fragment::AFTER: return std::move(ret) + "after(" + util::ToString(node.k) + ")"; + case Fragment::OLDER: return std::move(ret) + "older(" + util::ToString(node.k) + ")"; case Fragment::HASH256: return std::move(ret) + "hash256(" + HexStr(node.data) + ")"; case Fragment::HASH160: return std::move(ret) + "hash160(" + HexStr(node.data) + ")"; case Fragment::SHA256: return std::move(ret) + "sha256(" + HexStr(node.data) + ")"; @@ -883,7 +883,7 @@ public: return std::move(ret) + "andor(" + std::move(subs[0]) + "," + std::move(subs[1]) + "," + std::move(subs[2]) + ")"; case Fragment::MULTI: { CHECK_NONFATAL(!is_tapscript); - auto str = std::move(ret) + "multi(" + ::ToString(node.k); + auto str = std::move(ret) + "multi(" + util::ToString(node.k); for (const auto& key : node.keys) { auto key_str = ctx.ToString(key); if (!key_str) return {}; @@ -893,7 +893,7 @@ public: } case Fragment::MULTI_A: { CHECK_NONFATAL(is_tapscript); - auto str = std::move(ret) + "multi_a(" + ::ToString(node.k); + auto str = std::move(ret) + "multi_a(" + util::ToString(node.k); for (const auto& key : node.keys) { auto key_str = ctx.ToString(key); if (!key_str) return {}; @@ -902,7 +902,7 @@ public: return std::move(str) + ")"; } case Fragment::THRESH: { - auto str = std::move(ret) + "thresh(" + ::ToString(node.k); + auto str = std::move(ret) + "thresh(" + util::ToString(node.k); for (auto& sub : subs) { str += "," + std::move(sub); } @@ -1764,7 +1764,7 @@ void BuildBack(const MiniscriptContext script_ctx, Fragment nt, std::vector<Node template<typename Key, typename Ctx> inline NodeRef<Key> Parse(Span<const char> in, const Ctx& ctx) { - using namespace spanparsing; + using namespace script; // Account for the minimum script size for all parsed fragments so far. It "borrows" 1 // script byte from all leaf nodes, counting it instead whenever a space for a recursive diff --git a/src/util/spanparsing.cpp b/src/script/parsing.cpp index c464fc2b87..3528ac9bfa 100644 --- a/src/util/spanparsing.cpp +++ b/src/script/parsing.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <util/spanparsing.h> +#include <script/parsing.h> #include <span.h> @@ -10,7 +10,7 @@ #include <cstddef> #include <string> -namespace spanparsing { +namespace script { bool Const(const std::string& str, Span<const char>& sp) { @@ -49,4 +49,4 @@ Span<const char> Expr(Span<const char>& sp) return ret; } -} // namespace spanparsing +} // namespace script diff --git a/src/script/parsing.h b/src/script/parsing.h new file mode 100644 index 0000000000..850faea041 --- /dev/null +++ b/src/script/parsing.h @@ -0,0 +1,40 @@ +// Copyright (c) 2018-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_SCRIPT_PARSING_H +#define BITCOIN_SCRIPT_PARSING_H + +#include <span.h> + +#include <string> + +namespace script { + +/** Parse a constant. + * + * If sp's initial part matches str, sp is updated to skip that part, and true is returned. + * Otherwise sp is unmodified and false is returned. + */ +bool Const(const std::string& str, Span<const char>& sp); + +/** Parse a function call. + * + * If sp's initial part matches str + "(", and sp ends with ")", sp is updated to be the + * section between the braces, and true is returned. Otherwise sp is unmodified and false + * is returned. + */ +bool Func(const std::string& str, Span<const char>& sp); + +/** Extract the expression that sp begins with. + * + * This function will return the initial part of sp, up to (but not including) the first + * comma or closing brace, skipping ones that are surrounded by braces. So for example, + * for "foo(bar(1),2),3" the initial part "foo(bar(1),2)" will be returned. sp will be + * updated to skip the initial part that is returned. + */ +Span<const char> Expr(Span<const char>& sp); + +} // namespace script + +#endif // BITCOIN_SCRIPT_PARSING_H diff --git a/src/script/script.cpp b/src/script/script.cpp index 80e8d26bcf..73ea336c4f 100644 --- a/src/script/script.cpp +++ b/src/script/script.cpp @@ -6,10 +6,10 @@ #include <script/script.h> #include <crypto/common.h> +#include <crypto/hex_base.h> #include <hash.h> #include <uint256.h> #include <util/hash_type.h> -#include <util/strencodings.h> #include <string> diff --git a/src/signet.cpp b/src/signet.cpp index ebf0de09d3..7c193a1d77 100644 --- a/src/signet.cpp +++ b/src/signet.cpp @@ -68,13 +68,13 @@ static uint256 ComputeModifiedMerkleRoot(const CMutableTransaction& cb, const CB std::optional<SignetTxs> SignetTxs::Create(const CBlock& block, const CScript& challenge) { CMutableTransaction tx_to_spend; - tx_to_spend.nVersion = 0; + tx_to_spend.version = 0; tx_to_spend.nLockTime = 0; tx_to_spend.vin.emplace_back(COutPoint(), CScript(OP_0), 0); tx_to_spend.vout.emplace_back(0, challenge); CMutableTransaction tx_spending; - tx_spending.nVersion = 0; + tx_spending.version = 0; tx_spending.nLockTime = 0; tx_spending.vin.emplace_back(COutPoint(), CScript(), 0); tx_spending.vout.emplace_back(0, CScript(OP_RETURN)); diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 9668a85484..e5d25637bd 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -22,6 +22,7 @@ using namespace std::literals; using node::NodeContext; +using util::ToString; static NetGroupManager EMPTY_NETGROUPMAN{std::vector<bool>()}; static const bool DETERMINISTIC{true}; diff --git a/src/test/argsman_tests.cpp b/src/test/argsman_tests.cpp index 340208a1c9..5f0318e8c4 100644 --- a/src/test/argsman_tests.cpp +++ b/src/test/argsman_tests.cpp @@ -20,6 +20,8 @@ #include <boost/test/unit_test.hpp> +using util::ToString; + BOOST_FIXTURE_TEST_SUITE(argsman_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(util_datadir) diff --git a/src/test/blockchain_tests.cpp b/src/test/blockchain_tests.cpp index be515a9eac..9b8f419290 100644 --- a/src/test/blockchain_tests.cpp +++ b/src/test/blockchain_tests.cpp @@ -11,6 +11,8 @@ #include <cstdlib> +using util::ToString; + /* Equality between doubles is imprecise. Comparison should be done * with a small threshold of tolerance, rather than exact equality. */ diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp index efe0983698..9eb7acc3ca 100644 --- a/src/test/blockmanager_tests.cpp +++ b/src/test/blockmanager_tests.cpp @@ -27,7 +27,7 @@ BOOST_FIXTURE_TEST_SUITE(blockmanager_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos) { const auto params {CreateChainParams(ArgsManager{}, ChainType::MAIN)}; - KernelNotifications notifications{*Assert(m_node.shutdown), m_node.exit_status}; + KernelNotifications notifications{*Assert(m_node.shutdown), m_node.exit_status, *Assert(m_node.warnings)}; const BlockManager::Options blockman_opts{ .chainparams = *params, .blocks_dir = m_args.GetBlocksDirPath(), @@ -134,7 +134,7 @@ BOOST_FIXTURE_TEST_CASE(blockmanager_block_data_availability, TestChain100Setup) BOOST_AUTO_TEST_CASE(blockmanager_flush_block_file) { - KernelNotifications notifications{*Assert(m_node.shutdown), m_node.exit_status}; + KernelNotifications notifications{*Assert(m_node.shutdown), m_node.exit_status, *Assert(m_node.warnings)}; node::BlockManager::Options blockman_opts{ .chainparams = Params(), .blocks_dir = m_args.GetBlocksDirPath(), diff --git a/src/test/data/tx_invalid.json b/src/test/data/tx_invalid.json index a47bc8f366..486469ddef 100644 --- a/src/test/data/tx_invalid.json +++ b/src/test/data/tx_invalid.json @@ -246,7 +246,7 @@ [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0x7c17aff532f22beb54069942f9bf567a66133eaf EQUAL"]], "0200000001000100000000000000000000000000000000000000000000000000000000000000000000030251b2000000000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], -["Failure due to insufficient tx.nVersion (<2)"], +["Failure due to insufficient tx.version (<2)"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0 CHECKSEQUENCEVERIFY 1"]], "010000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "CHECKSEQUENCEVERIFY"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4194304 CHECKSEQUENCEVERIFY"]], diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index 723a1ceee3..167e4be288 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -12,6 +12,8 @@ #include <boost/test/unit_test.hpp> +using util::ToString; + // Test if a string consists entirely of null characters static bool is_null_key(const std::vector<unsigned char>& key) { bool isnull = true; diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index bee9695b08..42db28daf5 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -141,7 +141,7 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) { NodeId id{0}; auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman, Params()); - auto peerLogic = PeerManager::make(*connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, {}); + auto peerLogic = PeerManager::make(*connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, *m_node.warnings, {}); constexpr int max_outbound_full_relay = MAX_OUTBOUND_FULL_RELAY_CONNECTIONS; CConnman::Options options; @@ -239,7 +239,7 @@ BOOST_AUTO_TEST_CASE(block_relay_only_eviction) { NodeId id{0}; auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman, Params()); - auto peerLogic = PeerManager::make(*connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, {}); + auto peerLogic = PeerManager::make(*connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, *m_node.warnings, {}); constexpr int max_outbound_block_relay{MAX_BLOCK_RELAY_ONLY_CONNECTIONS}; constexpr int64_t MINIMUM_CONNECT_TIME{30}; @@ -300,7 +300,7 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) auto banman = std::make_unique<BanMan>(m_args.GetDataDirBase() / "banlist", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman, Params()); - auto peerLogic = PeerManager::make(*connman, *m_node.addrman, banman.get(), *m_node.chainman, *m_node.mempool, {}); + auto peerLogic = PeerManager::make(*connman, *m_node.addrman, banman.get(), *m_node.chainman, *m_node.mempool, *m_node.warnings, {}); CNetAddr tor_netaddr; BOOST_REQUIRE( @@ -401,7 +401,7 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) auto banman = std::make_unique<BanMan>(m_args.GetDataDirBase() / "banlist", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman, Params()); - auto peerLogic = PeerManager::make(*connman, *m_node.addrman, banman.get(), *m_node.chainman, *m_node.mempool, {}); + auto peerLogic = PeerManager::make(*connman, *m_node.addrman, banman.get(), *m_node.chainman, *m_node.mempool, *m_node.warnings, {}); banman->ClearBanned(); int64_t nStartTime = GetTime(); diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index c779bf6f73..e6821dd321 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -7,6 +7,7 @@ #include <script/sign.h> #include <test/util/setup_common.h> #include <util/strencodings.h> +#include <util/string.h> #include <boost/test/unit_test.hpp> @@ -14,6 +15,8 @@ #include <string> #include <vector> +using util::Split; + namespace { void CheckUnparsable(const std::string& prv, const std::string& pub, const std::string& expected_error) @@ -400,7 +403,6 @@ void CheckInferDescriptor(const std::string& script_hex, const std::string& expe provider.pubkeys.emplace(origin_pubkey.GetID(), origin_pubkey); if (!origin_str.empty()) { - using namespace spanparsing; KeyOriginInfo info; Span<const char> origin_sp{origin_str}; std::vector<Span<const char>> origin_split = Split(origin_sp, "/"); diff --git a/src/test/fuzz/base_encode_decode.cpp b/src/test/fuzz/base_encode_decode.cpp index d322416d34..0cc8cb5886 100644 --- a/src/test/fuzz/base_encode_decode.cpp +++ b/src/test/fuzz/base_encode_decode.cpp @@ -14,6 +14,9 @@ #include <string> #include <vector> +using util::TrimString; +using util::TrimStringView; + FUZZ_TARGET(base_encode_decode) { const std::string random_encoded_string(buffer.begin(), buffer.end()); diff --git a/src/test/fuzz/bitset.cpp b/src/test/fuzz/bitset.cpp new file mode 100644 index 0000000000..7684337729 --- /dev/null +++ b/src/test/fuzz/bitset.cpp @@ -0,0 +1,316 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <span.h> +#include <test/fuzz/util.h> +#include <test/util/xoroshiro128plusplus.h> +#include <util/bitset.h> + +#include <bitset> +#include <vector> + +namespace { + +/** Pop the first byte from a byte-span, and return it. */ +uint8_t ReadByte(FuzzBufferType& buffer) +{ + if (buffer.empty()) return 0; + uint8_t ret = buffer.front(); + buffer = buffer.subspan(1); + return ret; +} + +/** Perform a simulation fuzz test on BitSet type S. */ +template<typename S> +void TestType(FuzzBufferType buffer) +{ + /** This fuzz test's design is based on the assumption that the actual bits stored in the + * bitsets and their simulations do not matter for the purpose of detecting edge cases, thus + * these are taken from a deterministically-seeded RNG instead. To provide some level of + * variation however, pick the seed based on the buffer size and size of the chosen bitset. */ + XoRoShiRo128PlusPlus rng(buffer.size() + 0x10000 * S::Size()); + + using Sim = std::bitset<S::Size()>; + // Up to 4 real BitSets (initially 2). + std::vector<S> real(2); + // Up to 4 std::bitsets with the same corresponding contents. + std::vector<Sim> sim(2); + + /* Compare sim[idx] with real[idx], using all inspector operations. */ + auto compare_fn = [&](unsigned idx) { + /* iterators and operator[] */ + auto it = real[idx].begin(); + unsigned first = S::Size(); + unsigned last = S::Size(); + for (unsigned i = 0; i < S::Size(); ++i) { + bool match = (it != real[idx].end()) && *it == i; + assert(sim[idx][i] == real[idx][i]); + assert(match == real[idx][i]); + assert((it == real[idx].end()) != (it != real[idx].end())); + if (match) { + ++it; + if (first == S::Size()) first = i; + last = i; + } + } + assert(it == real[idx].end()); + assert(!(it != real[idx].end())); + /* Any / None */ + assert(sim[idx].any() == real[idx].Any()); + assert(sim[idx].none() == real[idx].None()); + /* First / Last */ + if (sim[idx].any()) { + assert(first == real[idx].First()); + assert(last == real[idx].Last()); + } + /* Count */ + assert(sim[idx].count() == real[idx].Count()); + }; + + LIMITED_WHILE(buffer.size() > 0, 1000) { + // Read one byte to determine which operation to execute on the BitSets. + int command = ReadByte(buffer) % 64; + // Read another byte that determines which bitsets will be involved. + unsigned args = ReadByte(buffer); + unsigned dest = ((args & 7) * sim.size()) >> 3; + unsigned src = (((args >> 3) & 7) * sim.size()) >> 3; + unsigned aux = (((args >> 6) & 3) * sim.size()) >> 2; + // Args are in range for non-empty sim, or sim is completely empty and will be grown + assert((sim.empty() && dest == 0 && src == 0 && aux == 0) || + (!sim.empty() && dest < sim.size() && src < sim.size() && aux < sim.size())); + + // Pick one operation based on value of command. Not all operations are always applicable. + // Loop through the applicable ones until command reaches 0 (which avoids the need to + // compute the number of applicable commands ahead of time). + while (true) { + if (dest < sim.size() && command-- == 0) { + /* Set() (true) */ + unsigned val = ReadByte(buffer) % S::Size(); + assert(sim[dest][val] == real[dest][val]); + sim[dest].set(val); + real[dest].Set(val); + break; + } else if (dest < sim.size() && command-- == 0) { + /* Reset() */ + unsigned val = ReadByte(buffer) % S::Size(); + assert(sim[dest][val] == real[dest][val]); + sim[dest].reset(val); + real[dest].Reset(val); + break; + } else if (dest < sim.size() && command-- == 0) { + /* Set() (conditional) */ + unsigned val = ReadByte(buffer) % S::Size(); + assert(sim[dest][val] == real[dest][val]); + sim[dest].set(val, args >> 7); + real[dest].Set(val, args >> 7); + break; + } else if (sim.size() < 4 && command-- == 0) { + /* Construct empty. */ + sim.resize(sim.size() + 1); + real.resize(real.size() + 1); + break; + } else if (sim.size() < 4 && command-- == 0) { + /* Construct singleton. */ + unsigned val = ReadByte(buffer) % S::Size(); + std::bitset<S::Size()> newset; + newset[val] = true; + sim.push_back(newset); + real.push_back(S::Singleton(val)); + break; + } else if (dest < sim.size() && command-- == 0) { + /* Make random. */ + compare_fn(dest); + sim[dest].reset(); + real[dest] = S{}; + for (unsigned i = 0; i < S::Size(); ++i) { + if (rng() & 1) { + sim[dest][i] = true; + real[dest].Set(i); + } + } + break; + } else if (dest < sim.size() && command-- == 0) { + /* Assign initializer list. */ + unsigned r1 = rng() % S::Size(); + unsigned r2 = rng() % S::Size(); + unsigned r3 = rng() % S::Size(); + compare_fn(dest); + sim[dest].reset(); + real[dest] = {r1, r2, r3}; + sim[dest].set(r1); + sim[dest].set(r2); + sim[dest].set(r3); + break; + } else if (!sim.empty() && command-- == 0) { + /* Destruct. */ + compare_fn(sim.size() - 1); + sim.pop_back(); + real.pop_back(); + break; + } else if (sim.size() < 4 && src < sim.size() && command-- == 0) { + /* Copy construct. */ + sim.emplace_back(sim[src]); + real.emplace_back(real[src]); + break; + } else if (src < sim.size() && dest < sim.size() && command-- == 0) { + /* Copy assign. */ + compare_fn(dest); + sim[dest] = sim[src]; + real[dest] = real[src]; + break; + } else if (src < sim.size() && dest < sim.size() && command-- == 0) { + /* swap() function. */ + swap(sim[dest], sim[src]); + swap(real[dest], real[src]); + break; + } else if (sim.size() < 4 && command-- == 0) { + /* Construct with initializer list. */ + unsigned r1 = rng() % S::Size(); + unsigned r2 = rng() % S::Size(); + sim.emplace_back(); + sim.back().set(r1); + sim.back().set(r2); + real.push_back(S{r1, r2}); + break; + } else if (dest < sim.size() && command-- == 0) { + /* Fill() + copy assign. */ + unsigned len = ReadByte(buffer) % S::Size(); + compare_fn(dest); + sim[dest].reset(); + for (unsigned i = 0; i < len; ++i) sim[dest][i] = true; + real[dest] = S::Fill(len); + break; + } else if (src < sim.size() && command-- == 0) { + /* Iterator copy based compare. */ + unsigned val = ReadByte(buffer) % S::Size(); + /* In a first loop, compare begin..end, and copy to it_copy at some point. */ + auto it = real[src].begin(), it_copy = it; + for (unsigned i = 0; i < S::Size(); ++i) { + if (i == val) it_copy = it; + bool match = (it != real[src].end()) && *it == i; + assert(match == sim[src][i]); + if (match) ++it; + } + assert(it == real[src].end()); + /* Then compare from the copied point again to end. */ + for (unsigned i = val; i < S::Size(); ++i) { + bool match = (it_copy != real[src].end()) && *it_copy == i; + assert(match == sim[src][i]); + if (match) ++it_copy; + } + assert(it_copy == real[src].end()); + break; + } else if (src < sim.size() && dest < sim.size() && command-- == 0) { + /* operator|= */ + compare_fn(dest); + sim[dest] |= sim[src]; + real[dest] |= real[src]; + break; + } else if (src < sim.size() && dest < sim.size() && command-- == 0) { + /* operator&= */ + compare_fn(dest); + sim[dest] &= sim[src]; + real[dest] &= real[src]; + break; + } else if (src < sim.size() && dest < sim.size() && command-- == 0) { + /* operator-= */ + compare_fn(dest); + sim[dest] &= ~sim[src]; + real[dest] -= real[src]; + break; + } else if (src < sim.size() && dest < sim.size() && command-- == 0) { + /* operator^= */ + compare_fn(dest); + sim[dest] ^= sim[src]; + real[dest] ^= real[src]; + break; + } else if (src < sim.size() && dest < sim.size() && aux < sim.size() && command-- == 0) { + /* operator| */ + compare_fn(dest); + sim[dest] = sim[src] | sim[aux]; + real[dest] = real[src] | real[aux]; + break; + } else if (src < sim.size() && dest < sim.size() && aux < sim.size() && command-- == 0) { + /* operator& */ + compare_fn(dest); + sim[dest] = sim[src] & sim[aux]; + real[dest] = real[src] & real[aux]; + break; + } else if (src < sim.size() && dest < sim.size() && aux < sim.size() && command-- == 0) { + /* operator- */ + compare_fn(dest); + sim[dest] = sim[src] & ~sim[aux]; + real[dest] = real[src] - real[aux]; + break; + } else if (src < sim.size() && dest < sim.size() && aux < sim.size() && command-- == 0) { + /* operator^ */ + compare_fn(dest); + sim[dest] = sim[src] ^ sim[aux]; + real[dest] = real[src] ^ real[aux]; + break; + } else if (src < sim.size() && aux < sim.size() && command-- == 0) { + /* IsSupersetOf() and IsSubsetOf() */ + bool is_superset = (sim[aux] & ~sim[src]).none(); + bool is_subset = (sim[src] & ~sim[aux]).none(); + assert(real[src].IsSupersetOf(real[aux]) == is_superset); + assert(real[src].IsSubsetOf(real[aux]) == is_subset); + assert(real[aux].IsSupersetOf(real[src]) == is_subset); + assert(real[aux].IsSubsetOf(real[src]) == is_superset); + break; + } else if (src < sim.size() && aux < sim.size() && command-- == 0) { + /* operator== and operator!= */ + assert((sim[src] == sim[aux]) == (real[src] == real[aux])); + assert((sim[src] != sim[aux]) == (real[src] != real[aux])); + break; + } else if (src < sim.size() && aux < sim.size() && command-- == 0) { + /* Overlaps() */ + assert((sim[src] & sim[aux]).any() == real[src].Overlaps(real[aux])); + assert((sim[src] & sim[aux]).any() == real[aux].Overlaps(real[src])); + break; + } + } + } + /* Fully compare the final state. */ + for (unsigned i = 0; i < sim.size(); ++i) { + compare_fn(i); + } +} + +} // namespace + +FUZZ_TARGET(bitset) +{ + unsigned typdat = ReadByte(buffer) % 8; + if (typdat == 0) { + /* 16 bits */ + TestType<bitset_detail::IntBitSet<uint16_t>>(buffer); + TestType<bitset_detail::MultiIntBitSet<uint16_t, 1>>(buffer); + } else if (typdat == 1) { + /* 32 bits */ + TestType<bitset_detail::MultiIntBitSet<uint16_t, 2>>(buffer); + TestType<bitset_detail::IntBitSet<uint32_t>>(buffer); + } else if (typdat == 2) { + /* 48 bits */ + TestType<bitset_detail::MultiIntBitSet<uint16_t, 3>>(buffer); + } else if (typdat == 3) { + /* 64 bits */ + TestType<bitset_detail::IntBitSet<uint64_t>>(buffer); + TestType<bitset_detail::MultiIntBitSet<uint64_t, 1>>(buffer); + TestType<bitset_detail::MultiIntBitSet<uint32_t, 2>>(buffer); + TestType<bitset_detail::MultiIntBitSet<uint16_t, 4>>(buffer); + } else if (typdat == 4) { + /* 96 bits */ + TestType<bitset_detail::MultiIntBitSet<uint32_t, 3>>(buffer); + } else if (typdat == 5) { + /* 128 bits */ + TestType<bitset_detail::MultiIntBitSet<uint64_t, 2>>(buffer); + TestType<bitset_detail::MultiIntBitSet<uint32_t, 4>>(buffer); + } else if (typdat == 6) { + /* 192 bits */ + TestType<bitset_detail::MultiIntBitSet<uint64_t, 3>>(buffer); + } else if (typdat == 7) { + /* 256 bits */ + TestType<bitset_detail::MultiIntBitSet<uint64_t, 4>>(buffer); + } +} diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index c9a3bc86ac..6623edcf99 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -316,7 +316,8 @@ FUZZ_TARGET_DESERIALIZE(blocktransactionsrequest_deserialize, { DeserializeFromFuzzingInput(buffer, btr); }) FUZZ_TARGET_DESERIALIZE(snapshotmetadata_deserialize, { - SnapshotMetadata snapshot_metadata; + auto msg_start = Params().MessageStart(); + SnapshotMetadata snapshot_metadata{msg_start}; DeserializeFromFuzzingInput(buffer, snapshot_metadata); }) FUZZ_TARGET_DESERIALIZE(uint160_deserialize, { diff --git a/src/test/fuzz/fees.cpp b/src/test/fuzz/fees.cpp index 38a8c6798e..5c760be13d 100644 --- a/src/test/fuzz/fees.cpp +++ b/src/test/fuzz/fees.cpp @@ -2,17 +2,19 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/messages.h> #include <consensus/amount.h> #include <policy/fees.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> -#include <util/fees.h> #include <cstdint> #include <string> #include <vector> +using common::StringForFeeReason; + FUZZ_TARGET(fees) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); diff --git a/src/test/fuzz/fuzz.h b/src/test/fuzz/fuzz.h index ca74d53de7..c468cd39e3 100644 --- a/src/test/fuzz/fuzz.h +++ b/src/test/fuzz/fuzz.h @@ -5,10 +5,9 @@ #ifndef BITCOIN_TEST_FUZZ_FUZZ_H #define BITCOIN_TEST_FUZZ_FUZZ_H -#include <span.h> - #include <cstdint> #include <functional> +#include <span> #include <string_view> /** @@ -23,7 +22,7 @@ #define LIMITED_WHILE(condition, limit) \ for (unsigned _count{limit}; (condition) && _count; --_count) -using FuzzBufferType = Span<const uint8_t>; +using FuzzBufferType = std::span<const uint8_t>; using TypeTestOneInput = std::function<void(FuzzBufferType)>; struct FuzzTargetOptions { diff --git a/src/test/fuzz/i2p.cpp b/src/test/fuzz/i2p.cpp new file mode 100644 index 0000000000..3af5bed30a --- /dev/null +++ b/src/test/fuzz/i2p.cpp @@ -0,0 +1,63 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <common/args.h> +#include <i2p.h> +#include <netaddress.h> +#include <netbase.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/fuzz/util/net.h> +#include <test/util/setup_common.h> +#include <util/fs_helpers.h> +#include <util/threadinterrupt.h> + +void initialize_i2p() +{ + static const auto testing_setup = MakeNoLogFileContext<>(); +} + +FUZZ_TARGET(i2p, .init = initialize_i2p) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + + SetMockTime(ConsumeTime(fuzzed_data_provider)); + + // Mock CreateSock() to create FuzzedSock. + auto CreateSockOrig = CreateSock; + CreateSock = [&fuzzed_data_provider](const sa_family_t&) { + return std::make_unique<FuzzedSock>(fuzzed_data_provider); + }; + + const fs::path private_key_path = gArgs.GetDataDirNet() / "fuzzed_i2p_private_key"; + const CService addr{in6_addr(IN6ADDR_LOOPBACK_INIT), 7656}; + const Proxy sam_proxy{addr, false}; + CThreadInterrupt interrupt; + + i2p::sam::Session session{private_key_path, sam_proxy, &interrupt}; + i2p::Connection conn; + + if (session.Listen(conn)) { + if (session.Accept(conn)) { + try { + (void)conn.sock->RecvUntilTerminator('\n', 10ms, interrupt, i2p::sam::MAX_MSG_SIZE); + } catch (const std::runtime_error&) { + } + } + } + + bool proxy_error; + + if (session.Connect(CService{}, conn, proxy_error)) { + try { + conn.sock->SendComplete("verack\n", 10ms, interrupt); + } catch (const std::runtime_error&) { + } + } + + fs::remove_all(private_key_path); + + CreateSock = CreateSockOrig; +} diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index db246bb84e..8f1d7b6d45 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -40,6 +40,8 @@ #include <set> #include <vector> +using util::ToString; + void initialize_integer() { SelectParams(ChainType::REGTEST); diff --git a/src/test/fuzz/key.cpp b/src/test/fuzz/key.cpp index d389a29575..82973803f8 100644 --- a/src/test/fuzz/key.cpp +++ b/src/test/fuzz/key.cpp @@ -78,16 +78,6 @@ FUZZ_TARGET(key, .init = initialize_key) assert(copied_key == key); } - { - CKey negated_key = key; - negated_key.Negate(); - assert(negated_key.IsValid()); - assert(!(negated_key == key)); - - negated_key.Negate(); - assert(negated_key == key); - } - const uint256 random_uint256 = Hash(buffer); { diff --git a/src/test/fuzz/kitchen_sink.cpp b/src/test/fuzz/kitchen_sink.cpp index 82f3a306c5..4468f358d9 100644 --- a/src/test/fuzz/kitchen_sink.cpp +++ b/src/test/fuzz/kitchen_sink.cpp @@ -2,13 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/messages.h> #include <merkleblock.h> +#include <node/types.h> #include <policy/fees.h> #include <rpc/util.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> -#include <util/error.h> #include <util/translation.h> #include <array> @@ -16,17 +17,15 @@ #include <optional> #include <vector> +using common::TransactionErrorString; +using node::TransactionError; + namespace { constexpr TransactionError ALL_TRANSACTION_ERROR[] = { - TransactionError::OK, TransactionError::MISSING_INPUTS, TransactionError::ALREADY_IN_CHAIN, - TransactionError::P2P_DISABLED, TransactionError::MEMPOOL_REJECTED, TransactionError::MEMPOOL_ERROR, - TransactionError::INVALID_PSBT, - TransactionError::PSBT_MISMATCH, - TransactionError::SIGHASH_MISMATCH, TransactionError::MAX_FEE_EXCEEDED, }; }; // namespace diff --git a/src/test/fuzz/locale.cpp b/src/test/fuzz/locale.cpp index 0f2985b504..68db842247 100644 --- a/src/test/fuzz/locale.cpp +++ b/src/test/fuzz/locale.cpp @@ -51,7 +51,7 @@ FUZZ_TARGET(locale) int64_t parseint64_out_without_locale; const bool parseint64_without_locale = ParseInt64(random_string, &parseint64_out_without_locale); const int64_t random_int64 = fuzzed_data_provider.ConsumeIntegral<int64_t>(); - const std::string tostring_without_locale = ToString(random_int64); + const std::string tostring_without_locale = util::ToString(random_int64); // The variable `random_int32` is no longer used, but the harness still needs to // consume the same data that it did previously to not invalidate existing seeds. const int32_t random_int32 = fuzzed_data_provider.ConsumeIntegral<int32_t>(); @@ -75,7 +75,7 @@ FUZZ_TARGET(locale) if (parseint64_without_locale) { assert(parseint64_out_without_locale == parseint64_out_with_locale); } - const std::string tostring_with_locale = ToString(random_int64); + const std::string tostring_with_locale = util::ToString(random_int64); assert(tostring_without_locale == tostring_with_locale); const std::string strprintf_int_with_locale = strprintf("%d", random_int64); assert(strprintf_int_without_locale == strprintf_int_with_locale); diff --git a/src/test/fuzz/message.cpp b/src/test/fuzz/message.cpp index 75baaa2754..6763206f72 100644 --- a/src/test/fuzz/message.cpp +++ b/src/test/fuzz/message.cpp @@ -3,12 +3,12 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> +#include <common/signmessage.h> #include <key_io.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <util/chaintype.h> -#include <util/message.h> #include <util/strencodings.h> #include <cassert> diff --git a/src/test/fuzz/mini_miner.cpp b/src/test/fuzz/mini_miner.cpp index 84f9bb4ad0..3a1663364f 100644 --- a/src/test/fuzz/mini_miner.cpp +++ b/src/test/fuzz/mini_miner.cpp @@ -7,11 +7,13 @@ #include <test/util/txmempool.h> #include <test/util/mining.h> -#include <node/mini_miner.h> #include <node/miner.h> +#include <node/mini_miner.h> #include <primitives/transaction.h> #include <random.h> #include <txmempool.h> +#include <util/check.h> +#include <util/translation.h> #include <deque> #include <vector> @@ -33,7 +35,9 @@ void initialize_miner() FUZZ_TARGET(mini_miner, .init = initialize_miner) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - CTxMemPool pool{CTxMemPool::Options{}}; + bilingual_str error; + CTxMemPool pool{CTxMemPool::Options{}, error}; + Assert(error.empty()); std::vector<COutPoint> outpoints; std::deque<COutPoint> available_coins = g_available_coins; LOCK2(::cs_main, pool.cs); @@ -109,7 +113,9 @@ FUZZ_TARGET(mini_miner, .init = initialize_miner) FUZZ_TARGET(mini_miner_selection, .init = initialize_miner) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - CTxMemPool pool{CTxMemPool::Options{}}; + bilingual_str error; + CTxMemPool pool{CTxMemPool::Options{}, error}; + Assert(error.empty()); // Make a copy to preserve determinism. std::deque<COutPoint> available_coins = g_available_coins; std::vector<CTransactionRef> transactions; diff --git a/src/test/fuzz/muhash.cpp b/src/test/fuzz/muhash.cpp index 8304e6fdb8..dd34c465ed 100644 --- a/src/test/fuzz/muhash.cpp +++ b/src/test/fuzz/muhash.cpp @@ -43,7 +43,19 @@ FUZZ_TARGET(muhash) }, [&] { // Test that dividing a MuHash by itself brings it back to it's initial state + + // See note about clang + self-assignment in test/uint256_tests.cpp + #if defined(__clang__) + # pragma clang diagnostic push + # pragma clang diagnostic ignored "-Wself-assign-overloaded" + #endif + muhash /= muhash; + + #if defined(__clang__) + # pragma clang diagnostic pop + #endif + muhash.Finalize(out); out2 = uint256S(initial_state_hash); }, diff --git a/src/test/fuzz/package_eval.cpp b/src/test/fuzz/package_eval.cpp index c201118bce..53aedf23ea 100644 --- a/src/test/fuzz/package_eval.cpp +++ b/src/test/fuzz/package_eval.cpp @@ -15,7 +15,9 @@ #include <test/util/script.h> #include <test/util/setup_common.h> #include <test/util/txmempool.h> +#include <util/check.h> #include <util/rbf.h> +#include <util/translation.h> #include <validation.h> #include <validationinterface.h> @@ -107,7 +109,7 @@ void MockTime(FuzzedDataProvider& fuzzed_data_provider, const Chainstate& chains SetMockTime(time); } -CTxMemPool MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeContext& node) +std::unique_ptr<CTxMemPool> MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeContext& node) { // Take the default options for tests... CTxMemPool::Options mempool_opts{MemPoolOptionsForTest(node)}; @@ -126,8 +128,13 @@ CTxMemPool MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeConte mempool_opts.check_ratio = 1; mempool_opts.require_standard = fuzzed_data_provider.ConsumeBool(); + bilingual_str error; // ...and construct a CTxMemPool from it - return CTxMemPool{mempool_opts}; + auto mempool{std::make_unique<CTxMemPool>(std::move(mempool_opts), error)}; + // ... ignore the error since it might be beneficial to fuzz even when the + // mempool size is unreasonably small + Assert(error.empty() || error.original.starts_with("-maxmempool must be at least ")); + return mempool; } FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) @@ -149,8 +156,8 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) auto outpoints_updater = std::make_shared<OutpointsUpdater>(mempool_outpoints); node.validation_signals->RegisterSharedValidationInterface(outpoints_updater); - CTxMemPool tx_pool_{MakeMempool(fuzzed_data_provider, node)}; - MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(&tx_pool_); + auto tx_pool_{MakeMempool(fuzzed_data_provider, node)}; + MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(tx_pool_.get()); chainstate.SetMempool(&tx_pool); @@ -173,7 +180,7 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) // Create transaction to add to the mempool const CTransactionRef tx = [&] { CMutableTransaction tx_mut; - tx_mut.nVersion = fuzzed_data_provider.ConsumeBool() ? 3 : CTransaction::CURRENT_VERSION; + tx_mut.version = fuzzed_data_provider.ConsumeBool() ? TRUC_VERSION : CTransaction::CURRENT_VERSION; tx_mut.nLockTime = fuzzed_data_provider.ConsumeBool() ? 0 : fuzzed_data_provider.ConsumeIntegral<uint32_t>(); // Last tx will sweep all outpoints in package const auto num_in = last_tx ? package_outpoints.size() : fuzzed_data_provider.ConsumeIntegralInRange<int>(1, mempool_outpoints.size()); @@ -307,7 +314,7 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) // just use result_package.m_state here. This makes the expect_valid check meaningless, but // we can still verify that the contents of m_tx_results are consistent with m_state. const bool expect_valid{result_package.m_state.IsValid()}; - Assert(!CheckPackageMempoolAcceptResult(txs, result_package, expect_valid, nullptr)); + Assert(!CheckPackageMempoolAcceptResult(txs, result_package, expect_valid, &tx_pool)); } else { // This is empty if it fails early checks, or "full" if transactions are looked at deeper Assert(result_package.m_tx_results.size() == txs.size() || result_package.m_tx_results.empty()); diff --git a/src/test/fuzz/partially_downloaded_block.cpp b/src/test/fuzz/partially_downloaded_block.cpp index 2bf47930f4..791d457710 100644 --- a/src/test/fuzz/partially_downloaded_block.cpp +++ b/src/test/fuzz/partially_downloaded_block.cpp @@ -10,6 +10,8 @@ #include <test/util/setup_common.h> #include <test/util/txmempool.h> #include <txmempool.h> +#include <util/check.h> +#include <util/translation.h> #include <cstddef> #include <cstdint> @@ -52,7 +54,9 @@ FUZZ_TARGET(partially_downloaded_block, .init = initialize_pdb) CBlockHeaderAndShortTxIDs cmpctblock{*block}; - CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; + bilingual_str error; + CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node), error}; + Assert(error.empty()); PartiallyDownloadedBlock pdb{&pool}; // Set of available transactions (mempool or extra_txn) diff --git a/src/test/fuzz/rbf.cpp b/src/test/fuzz/rbf.cpp index 64785948f6..eb981352ec 100644 --- a/src/test/fuzz/rbf.cpp +++ b/src/test/fuzz/rbf.cpp @@ -13,6 +13,8 @@ #include <test/util/setup_common.h> #include <test/util/txmempool.h> #include <txmempool.h> +#include <util/check.h> +#include <util/translation.h> #include <cstdint> #include <optional> @@ -56,7 +58,9 @@ FUZZ_TARGET(rbf, .init = initialize_rbf) return; } - CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; + bilingual_str error; + CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node), error}; + Assert(error.empty()); LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), NUM_ITERS) { @@ -87,10 +91,14 @@ FUZZ_TARGET(package_rbf, .init = initialize_package_rbf) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); SetMockTime(ConsumeTime(fuzzed_data_provider)); - std::optional<CMutableTransaction> child = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); - if (!child) return; + // "Real" virtual size is not important for this test since ConsumeTxMemPoolEntry generates its own virtual size values + // so we construct small transactions for performance reasons. Child simply needs an input for later to perhaps connect to parent. + CMutableTransaction child; + child.vin.resize(1); - CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; + bilingual_str error; + CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node), error}; + Assert(error.empty()); // Add a bunch of parent-child pairs to the mempool, and remember them. std::vector<CTransaction> mempool_txs; @@ -107,15 +115,13 @@ FUZZ_TARGET(package_rbf, .init = initialize_package_rbf) LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), NUM_ITERS) { // Make sure txns only have one input, and that a unique input is given to avoid circular references - std::optional<CMutableTransaction> parent = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); - if (!parent) { - return; - } + CMutableTransaction parent; assert(iter <= g_outpoints.size()); - parent->vin.resize(1); - parent->vin[0].prevout = g_outpoints[iter++]; + parent.vin.resize(1); + parent.vin[0].prevout = g_outpoints[iter++]; + parent.vout.emplace_back(0, CScript()); - mempool_txs.emplace_back(*parent); + mempool_txs.emplace_back(parent); const auto parent_entry = ConsumeTxMemPoolEntry(fuzzed_data_provider, mempool_txs.back()); running_vsize_total += parent_entry.GetTxSize(); if (running_vsize_total > std::numeric_limits<int32_t>::max()) { @@ -124,10 +130,10 @@ FUZZ_TARGET(package_rbf, .init = initialize_package_rbf) break; } pool.addUnchecked(parent_entry); - if (fuzzed_data_provider.ConsumeBool() && !child->vin.empty()) { - child->vin[0].prevout = COutPoint{mempool_txs.back().GetHash(), 0}; + if (fuzzed_data_provider.ConsumeBool()) { + child.vin[0].prevout = COutPoint{mempool_txs.back().GetHash(), 0}; } - mempool_txs.emplace_back(*child); + mempool_txs.emplace_back(child); const auto child_entry = ConsumeTxMemPoolEntry(fuzzed_data_provider, mempool_txs.back()); running_vsize_total += child_entry.GetTxSize(); if (running_vsize_total > std::numeric_limits<int32_t>::max()) { diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 2325bf0941..4e52c1c091 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -36,6 +36,9 @@ #include <vector> enum class ChainType; +using util::Join; +using util::ToString; + namespace { struct RPCFuzzTestingSetup : public TestingSetup { RPCFuzzTestingSetup(const ChainType chain_type, const std::vector<const char*>& extra_args) : TestingSetup{chain_type, extra_args} diff --git a/src/test/fuzz/script_assets_test_minimizer.cpp b/src/test/fuzz/script_assets_test_minimizer.cpp index 511b581f60..5a8b599df6 100644 --- a/src/test/fuzz/script_assets_test_minimizer.cpp +++ b/src/test/fuzz/script_assets_test_minimizer.cpp @@ -17,6 +17,8 @@ #include <string> #include <vector> +using util::SplitString; + // This fuzz "test" can be used to minimize test cases for script_assets_test in // src/test/script_tests.cpp. While it written as a fuzz test, and can be used as such, // fuzzing the inputs is unlikely to construct useful test cases. diff --git a/src/test/fuzz/spanparsing.cpp b/src/test/fuzz/script_parsing.cpp index b8996632bc..d29a6ea90c 100644 --- a/src/test/fuzz/spanparsing.cpp +++ b/src/test/fuzz/script_parsing.cpp @@ -2,11 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <script/parsing.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> -#include <util/spanparsing.h> +#include <util/string.h> -FUZZ_TARGET(spanparsing) +using util::Split; + +FUZZ_TARGET(script_parsing) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const size_t query_size = fuzzed_data_provider.ConsumeIntegral<size_t>(); @@ -15,16 +18,16 @@ FUZZ_TARGET(spanparsing) const Span<const char> const_span{span_str}; Span<const char> mut_span = const_span; - (void)spanparsing::Const(query, mut_span); + (void)script::Const(query, mut_span); mut_span = const_span; - (void)spanparsing::Func(query, mut_span); + (void)script::Func(query, mut_span); mut_span = const_span; - (void)spanparsing::Expr(mut_span); + (void)script::Expr(mut_span); if (!query.empty()) { mut_span = const_span; - (void)spanparsing::Split(mut_span, query.front()); + (void)Split(mut_span, query.front()); } } diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index 631da13803..5b822b03f6 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -5,6 +5,7 @@ #include <blockfilter.h> #include <clientversion.h> #include <common/args.h> +#include <common/messages.h> #include <common/settings.h> #include <common/system.h> #include <common/url.h> @@ -21,8 +22,6 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> -#include <util/error.h> -#include <util/fees.h> #include <util/strencodings.h> #include <util/string.h> #include <util/translation.h> @@ -37,6 +36,16 @@ enum class FeeEstimateMode; +using common::AmountErrMsg; +using common::AmountHighWarn; +using common::FeeModeFromString; +using common::ResolveErrMsg; +using util::ContainsNoNUL; +using util::Join; +using util::RemovePrefix; +using util::SplitString; +using util::TrimString; + FUZZ_TARGET(string) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); diff --git a/src/test/fuzz/timeoffsets.cpp b/src/test/fuzz/timeoffsets.cpp index 019337a94a..dfa4dd705d 100644 --- a/src/test/fuzz/timeoffsets.cpp +++ b/src/test/fuzz/timeoffsets.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <node/timeoffsets.h> +#include <node/warnings.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/util/setup_common.h> @@ -19,7 +20,8 @@ void initialize_timeoffsets() FUZZ_TARGET(timeoffsets, .init = initialize_timeoffsets) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - TimeOffsets offsets{}; + node::Warnings warnings{}; + TimeOffsets offsets{warnings}; LIMITED_WHILE(fuzzed_data_provider.remaining_bytes() > 0, 4'000) { (void)offsets.Median(); offsets.Add(std::chrono::seconds{fuzzed_data_provider.ConsumeIntegral<std::chrono::seconds::rep>()}); diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index 9f0aedf29b..b6b91445f9 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -15,12 +15,15 @@ #include <test/util/script.h> #include <test/util/setup_common.h> #include <test/util/txmempool.h> +#include <util/check.h> #include <util/rbf.h> +#include <util/translation.h> #include <validation.h> #include <validationinterface.h> using node::BlockAssembler; using node::NodeContext; +using util::ToString; namespace { @@ -116,7 +119,7 @@ void MockTime(FuzzedDataProvider& fuzzed_data_provider, const Chainstate& chains SetMockTime(time); } -CTxMemPool MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeContext& node) +std::unique_ptr<CTxMemPool> MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeContext& node) { // Take the default options for tests... CTxMemPool::Options mempool_opts{MemPoolOptionsForTest(node)}; @@ -126,7 +129,12 @@ CTxMemPool MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeConte mempool_opts.require_standard = fuzzed_data_provider.ConsumeBool(); // ...and construct a CTxMemPool from it - return CTxMemPool{mempool_opts}; + bilingual_str error; + auto mempool{std::make_unique<CTxMemPool>(std::move(mempool_opts), error)}; + // ... ignore the error since it might be beneficial to fuzz even when the + // mempool size is unreasonably small + Assert(error.empty() || error.original.starts_with("-maxmempool must be at least ")); + return mempool; } void CheckATMPInvariants(const MempoolAcceptResult& res, bool txid_in_mempool, bool wtxid_in_mempool) @@ -198,8 +206,8 @@ FUZZ_TARGET(tx_pool_standard, .init = initialize_tx_pool) constexpr CAmount SUPPLY_TOTAL{COINBASE_MATURITY * 50 * COIN}; SetMempoolConstraints(*node.args, fuzzed_data_provider); - CTxMemPool tx_pool_{MakeMempool(fuzzed_data_provider, node)}; - MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(&tx_pool_); + auto tx_pool_{MakeMempool(fuzzed_data_provider, node)}; + MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(tx_pool_.get()); chainstate.SetMempool(&tx_pool); @@ -226,7 +234,7 @@ FUZZ_TARGET(tx_pool_standard, .init = initialize_tx_pool) // Create transaction to add to the mempool const CTransactionRef tx = [&] { CMutableTransaction tx_mut; - tx_mut.nVersion = fuzzed_data_provider.ConsumeBool() ? 3 : CTransaction::CURRENT_VERSION; + tx_mut.version = fuzzed_data_provider.ConsumeBool() ? TRUC_VERSION : CTransaction::CURRENT_VERSION; tx_mut.nLockTime = fuzzed_data_provider.ConsumeBool() ? 0 : fuzzed_data_provider.ConsumeIntegral<uint32_t>(); const auto num_in = fuzzed_data_provider.ConsumeIntegralInRange<int>(1, outpoints_rbf.size()); const auto num_out = fuzzed_data_provider.ConsumeIntegralInRange<int>(1, outpoints_rbf.size() * 2); @@ -376,8 +384,8 @@ FUZZ_TARGET(tx_pool, .init = initialize_tx_pool) } SetMempoolConstraints(*node.args, fuzzed_data_provider); - CTxMemPool tx_pool_{MakeMempool(fuzzed_data_provider, node)}; - MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(&tx_pool_); + auto tx_pool_{MakeMempool(fuzzed_data_provider, node)}; + MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(tx_pool_.get()); chainstate.SetMempool(&tx_pool); diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index 259b00fcae..92ded99917 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -43,9 +43,9 @@ CMutableTransaction ConsumeTransaction(FuzzedDataProvider& fuzzed_data_provider, { CMutableTransaction tx_mut; const auto p2wsh_op_true = fuzzed_data_provider.ConsumeBool(); - tx_mut.nVersion = fuzzed_data_provider.ConsumeBool() ? + tx_mut.version = fuzzed_data_provider.ConsumeBool() ? CTransaction::CURRENT_VERSION : - fuzzed_data_provider.ConsumeIntegral<int32_t>(); + fuzzed_data_provider.ConsumeIntegral<uint32_t>(); tx_mut.nLockTime = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); const auto num_in = fuzzed_data_provider.ConsumeIntegralInRange<int>(0, max_num_in); const auto num_out = fuzzed_data_provider.ConsumeIntegralInRange<int>(0, max_num_out); diff --git a/src/test/fuzz/util/net.cpp b/src/test/fuzz/util/net.cpp index 99151bb84d..5e7aae670e 100644 --- a/src/test/fuzz/util/net.cpp +++ b/src/test/fuzz/util/net.cpp @@ -193,21 +193,20 @@ ssize_t FuzzedSock::Recv(void* buf, size_t len, int flags) const bool pad_to_len_bytes{m_fuzzed_data_provider.ConsumeBool()}; if (m_peek_data.has_value()) { // `MSG_PEEK` was used in the preceding `Recv()` call, return `m_peek_data`. - random_bytes.assign({m_peek_data.value()}); + random_bytes = m_peek_data.value(); if ((flags & MSG_PEEK) == 0) { m_peek_data.reset(); } pad_to_len_bytes = false; } else if ((flags & MSG_PEEK) != 0) { // New call with `MSG_PEEK`. - random_bytes = m_fuzzed_data_provider.ConsumeBytes<uint8_t>(1); + random_bytes = ConsumeRandomLengthByteVector(m_fuzzed_data_provider, len); if (!random_bytes.empty()) { - m_peek_data = random_bytes[0]; + m_peek_data = random_bytes; pad_to_len_bytes = false; } } else { - random_bytes = m_fuzzed_data_provider.ConsumeBytes<uint8_t>( - m_fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, len)); + random_bytes = ConsumeRandomLengthByteVector(m_fuzzed_data_provider, len); } if (random_bytes.empty()) { const ssize_t r = m_fuzzed_data_provider.ConsumeBool() ? 0 : -1; @@ -216,7 +215,11 @@ ssize_t FuzzedSock::Recv(void* buf, size_t len, int flags) const } return r; } - std::memcpy(buf, random_bytes.data(), random_bytes.size()); + // `random_bytes` might exceed the size of `buf` if e.g. Recv is called with + // len=N and MSG_PEEK first and afterwards called with len=M (M < N) and + // without MSG_PEEK. + size_t recv_len{std::min(random_bytes.size(), len)}; + std::memcpy(buf, random_bytes.data(), recv_len); if (pad_to_len_bytes) { if (len > random_bytes.size()) { std::memset((char*)buf + random_bytes.size(), 0, len - random_bytes.size()); @@ -226,7 +229,7 @@ ssize_t FuzzedSock::Recv(void* buf, size_t len, int flags) const if (m_fuzzed_data_provider.ConsumeBool() && std::getenv("FUZZED_SOCKET_FAKE_LATENCY") != nullptr) { std::this_thread::sleep_for(std::chrono::milliseconds{2}); } - return random_bytes.size(); + return recv_len; } int FuzzedSock::Connect(const sockaddr*, socklen_t) const @@ -380,7 +383,10 @@ bool FuzzedSock::Wait(std::chrono::milliseconds timeout, Event requested, Event* return false; } if (occurred != nullptr) { - *occurred = m_fuzzed_data_provider.ConsumeBool() ? requested : 0; + // We simulate the requested event as occured when ConsumeBool() + // returns false. This avoids simulating endless waiting if the + // FuzzedDataProvider runs out of data. + *occurred = m_fuzzed_data_provider.ConsumeBool() ? 0 : requested; } return true; } @@ -389,7 +395,10 @@ bool FuzzedSock::WaitMany(std::chrono::milliseconds timeout, EventsPerSock& even { for (auto& [sock, events] : events_per_sock) { (void)sock; - events.occurred = m_fuzzed_data_provider.ConsumeBool() ? events.requested : 0; + // We simulate the requested event as occured when ConsumeBool() + // returns false. This avoids simulating endless waiting if the + // FuzzedDataProvider runs out of data. + events.occurred = m_fuzzed_data_provider.ConsumeBool() ? 0 : events.requested; } return true; } diff --git a/src/test/fuzz/util/net.h b/src/test/fuzz/util/net.h index a6c9e23f2e..ed02680676 100644 --- a/src/test/fuzz/util/net.h +++ b/src/test/fuzz/util/net.h @@ -43,7 +43,7 @@ class FuzzedSock : public Sock * If `MSG_PEEK` is used, then our `Recv()` returns some random data as usual, but on the next * `Recv()` call we must return the same data, thus we remember it here. */ - mutable std::optional<uint8_t> m_peek_data; + mutable std::optional<std::vector<uint8_t>> m_peek_data; /** * Whether to pretend that the socket is select(2)-able. This is randomly set in the diff --git a/src/test/fuzz/utxo_snapshot.cpp b/src/test/fuzz/utxo_snapshot.cpp index dce728d96b..8c9c67a91c 100644 --- a/src/test/fuzz/utxo_snapshot.cpp +++ b/src/test/fuzz/utxo_snapshot.cpp @@ -47,7 +47,8 @@ FUZZ_TARGET(utxo_snapshot, .init = initialize_chain) const auto ActivateFuzzedSnapshot{[&] { AutoFile infile{fsbridge::fopen(snapshot_path, "rb")}; - SnapshotMetadata metadata; + auto msg_start = Params().MessageStart(); + SnapshotMetadata metadata{msg_start}; try { infile >> metadata; } catch (const std::ios_base::failure&) { diff --git a/src/test/fuzz/validation_load_mempool.cpp b/src/test/fuzz/validation_load_mempool.cpp index 00678742c9..51140ae039 100644 --- a/src/test/fuzz/validation_load_mempool.cpp +++ b/src/test/fuzz/validation_load_mempool.cpp @@ -13,7 +13,9 @@ #include <test/util/setup_common.h> #include <test/util/txmempool.h> #include <txmempool.h> +#include <util/check.h> #include <util/time.h> +#include <util/translation.h> #include <validation.h> #include <cstdint> @@ -40,7 +42,9 @@ FUZZ_TARGET(validation_load_mempool, .init = initialize_validation_load_mempool) SetMockTime(ConsumeTime(fuzzed_data_provider)); FuzzedFileProvider fuzzed_file_provider{fuzzed_data_provider}; - CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; + bilingual_str error; + CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node), error}; + Assert(error.empty()); auto& chainstate{static_cast<DummyChainState&>(g_setup->m_node.chainman->ActiveChainstate())}; chainstate.SetMempool(&pool); diff --git a/src/test/fuzz/vecdeque.cpp b/src/test/fuzz/vecdeque.cpp new file mode 100644 index 0000000000..1d9a98931f --- /dev/null +++ b/src/test/fuzz/vecdeque.cpp @@ -0,0 +1,491 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <span.h> +#include <test/fuzz/util.h> +#include <test/util/xoroshiro128plusplus.h> +#include <util/vecdeque.h> + +#include <deque> +#include <stdint.h> + +namespace { + +/** The maximum number of simultaneous buffers kept by the test. */ +static constexpr size_t MAX_BUFFERS{3}; +/** How many elements are kept in a buffer at most. */ +static constexpr size_t MAX_BUFFER_SIZE{48}; +/** How many operations are performed at most on the buffers in one test. */ +static constexpr size_t MAX_OPERATIONS{1024}; + +/** Perform a simulation fuzz test on VecDeque type T. + * + * T must be constructible from a uint64_t seed, comparable to other T, copyable, and movable. + */ +template<typename T, bool CheckNoneLeft> +void TestType(Span<const uint8_t> buffer, uint64_t rng_tweak) +{ + FuzzedDataProvider provider(buffer.data(), buffer.size()); + // Local RNG, only used for the seeds to initialize T objects with. + XoRoShiRo128PlusPlus rng(provider.ConsumeIntegral<uint64_t>() ^ rng_tweak); + + // Real circular buffers. + std::vector<VecDeque<T>> real; + real.reserve(MAX_BUFFERS); + // Simulated circular buffers. + std::vector<std::deque<T>> sim; + sim.reserve(MAX_BUFFERS); + // Temporary object of type T. + std::optional<T> tmp; + + // Compare a real and a simulated buffer. + auto compare_fn = [](const VecDeque<T>& r, const std::deque<T>& s) { + assert(r.size() == s.size()); + assert(r.empty() == s.empty()); + assert(r.capacity() >= r.size()); + if (s.size() == 0) return; + assert(r.front() == s.front()); + assert(r.back() == s.back()); + for (size_t i = 0; i < s.size(); ++i) { + assert(r[i] == s[i]); + } + }; + + LIMITED_WHILE(provider.remaining_bytes(), MAX_OPERATIONS) { + int command = provider.ConsumeIntegral<uint8_t>() % 64; + unsigned idx = real.empty() ? 0 : provider.ConsumeIntegralInRange<unsigned>(0, real.size() - 1); + const size_t num_buffers = sim.size(); + // Pick one operation based on value of command. Not all operations are always applicable. + // Loop through the applicable ones until command reaches 0 (which avoids the need to + // compute the number of applicable commands ahead of time). + const bool non_empty{num_buffers != 0}; + const bool non_full{num_buffers < MAX_BUFFERS}; + const bool partially_full{non_empty && non_full}; + const bool multiple_exist{num_buffers > 1}; + const bool existing_buffer_non_full{non_empty && sim[idx].size() < MAX_BUFFER_SIZE}; + const bool existing_buffer_non_empty{non_empty && !sim[idx].empty()}; + assert(non_full || non_empty); + while (true) { + if (non_full && command-- == 0) { + /* Default construct. */ + real.emplace_back(); + sim.emplace_back(); + break; + } + if (non_empty && command-- == 0) { + /* resize() */ + compare_fn(real[idx], sim[idx]); + size_t new_size = provider.ConsumeIntegralInRange<size_t>(0, MAX_BUFFER_SIZE); + real[idx].resize(new_size); + sim[idx].resize(new_size); + assert(real[idx].size() == new_size); + break; + } + if (non_empty && command-- == 0) { + /* clear() */ + compare_fn(real[idx], sim[idx]); + real[idx].clear(); + sim[idx].clear(); + assert(real[idx].empty()); + break; + } + if (non_empty && command-- == 0) { + /* Copy construct default. */ + compare_fn(real[idx], sim[idx]); + real[idx] = VecDeque<T>(); + sim[idx].clear(); + assert(real[idx].size() == 0); + break; + } + if (non_empty && command-- == 0) { + /* Destruct. */ + compare_fn(real.back(), sim.back()); + real.pop_back(); + sim.pop_back(); + break; + } + if (partially_full && command-- == 0) { + /* Copy construct. */ + real.emplace_back(real[idx]); + sim.emplace_back(sim[idx]); + break; + } + if (partially_full && command-- == 0) { + /* Move construct. */ + VecDeque<T> copy(real[idx]); + real.emplace_back(std::move(copy)); + sim.emplace_back(sim[idx]); + break; + } + if (multiple_exist && command-- == 0) { + /* swap() */ + swap(real[idx], real[(idx + 1) % num_buffers]); + swap(sim[idx], sim[(idx + 1) % num_buffers]); + break; + } + if (multiple_exist && command-- == 0) { + /* Copy assign. */ + compare_fn(real[idx], sim[idx]); + real[idx] = real[(idx + 1) % num_buffers]; + sim[idx] = sim[(idx + 1) % num_buffers]; + break; + } + if (multiple_exist && command-- == 0) { + /* Move assign. */ + VecDeque<T> copy(real[(idx + 1) % num_buffers]); + compare_fn(real[idx], sim[idx]); + real[idx] = std::move(copy); + sim[idx] = sim[(idx + 1) % num_buffers]; + break; + } + if (non_empty && command-- == 0) { + /* Self swap() */ + swap(real[idx], real[idx]); + break; + } + if (non_empty && command-- == 0) { + /* Self-copy assign. */ + real[idx] = real[idx]; + break; + } + if (non_empty && command-- == 0) { + /* Self-move assign. */ + // Do not use std::move(real[idx]) here: -Wself-move correctly warns about that. + real[idx] = static_cast<VecDeque<T>&&>(real[idx]); + break; + } + if (non_empty && command-- == 0) { + /* reserve() */ + size_t res_size = provider.ConsumeIntegralInRange<size_t>(0, MAX_BUFFER_SIZE); + size_t old_cap = real[idx].capacity(); + size_t old_size = real[idx].size(); + real[idx].reserve(res_size); + assert(real[idx].size() == old_size); + assert(real[idx].capacity() == std::max(old_cap, res_size)); + break; + } + if (non_empty && command-- == 0) { + /* shrink_to_fit() */ + size_t old_size = real[idx].size(); + real[idx].shrink_to_fit(); + assert(real[idx].size() == old_size); + assert(real[idx].capacity() == old_size); + break; + } + if (existing_buffer_non_full && command-- == 0) { + /* push_back() (copying) */ + tmp = T(rng()); + size_t old_size = real[idx].size(); + size_t old_cap = real[idx].capacity(); + real[idx].push_back(*tmp); + sim[idx].push_back(*tmp); + assert(real[idx].size() == old_size + 1); + if (old_cap > old_size) { + assert(real[idx].capacity() == old_cap); + } else { + assert(real[idx].capacity() > old_cap); + assert(real[idx].capacity() <= 2 * (old_cap + 1)); + } + break; + } + if (existing_buffer_non_full && command-- == 0) { + /* push_back() (moving) */ + tmp = T(rng()); + size_t old_size = real[idx].size(); + size_t old_cap = real[idx].capacity(); + sim[idx].push_back(*tmp); + real[idx].push_back(std::move(*tmp)); + assert(real[idx].size() == old_size + 1); + if (old_cap > old_size) { + assert(real[idx].capacity() == old_cap); + } else { + assert(real[idx].capacity() > old_cap); + assert(real[idx].capacity() <= 2 * (old_cap + 1)); + } + break; + } + if (existing_buffer_non_full && command-- == 0) { + /* emplace_back() */ + uint64_t seed{rng()}; + size_t old_size = real[idx].size(); + size_t old_cap = real[idx].capacity(); + sim[idx].emplace_back(seed); + real[idx].emplace_back(seed); + assert(real[idx].size() == old_size + 1); + if (old_cap > old_size) { + assert(real[idx].capacity() == old_cap); + } else { + assert(real[idx].capacity() > old_cap); + assert(real[idx].capacity() <= 2 * (old_cap + 1)); + } + break; + } + if (existing_buffer_non_full && command-- == 0) { + /* push_front() (copying) */ + tmp = T(rng()); + size_t old_size = real[idx].size(); + size_t old_cap = real[idx].capacity(); + real[idx].push_front(*tmp); + sim[idx].push_front(*tmp); + assert(real[idx].size() == old_size + 1); + if (old_cap > old_size) { + assert(real[idx].capacity() == old_cap); + } else { + assert(real[idx].capacity() > old_cap); + assert(real[idx].capacity() <= 2 * (old_cap + 1)); + } + break; + } + if (existing_buffer_non_full && command-- == 0) { + /* push_front() (moving) */ + tmp = T(rng()); + size_t old_size = real[idx].size(); + size_t old_cap = real[idx].capacity(); + sim[idx].push_front(*tmp); + real[idx].push_front(std::move(*tmp)); + assert(real[idx].size() == old_size + 1); + if (old_cap > old_size) { + assert(real[idx].capacity() == old_cap); + } else { + assert(real[idx].capacity() > old_cap); + assert(real[idx].capacity() <= 2 * (old_cap + 1)); + } + break; + } + if (existing_buffer_non_full && command-- == 0) { + /* emplace_front() */ + uint64_t seed{rng()}; + size_t old_size = real[idx].size(); + size_t old_cap = real[idx].capacity(); + sim[idx].emplace_front(seed); + real[idx].emplace_front(seed); + assert(real[idx].size() == old_size + 1); + if (old_cap > old_size) { + assert(real[idx].capacity() == old_cap); + } else { + assert(real[idx].capacity() > old_cap); + assert(real[idx].capacity() <= 2 * (old_cap + 1)); + } + break; + } + if (existing_buffer_non_empty && command-- == 0) { + /* front() [modifying] */ + tmp = T(rng()); + size_t old_size = real[idx].size(); + assert(sim[idx].front() == real[idx].front()); + sim[idx].front() = *tmp; + real[idx].front() = std::move(*tmp); + assert(real[idx].size() == old_size); + break; + } + if (existing_buffer_non_empty && command-- == 0) { + /* back() [modifying] */ + tmp = T(rng()); + size_t old_size = real[idx].size(); + assert(sim[idx].back() == real[idx].back()); + sim[idx].back() = *tmp; + real[idx].back() = *tmp; + assert(real[idx].size() == old_size); + break; + } + if (existing_buffer_non_empty && command-- == 0) { + /* operator[] [modifying] */ + tmp = T(rng()); + size_t pos = provider.ConsumeIntegralInRange<size_t>(0, sim[idx].size() - 1); + size_t old_size = real[idx].size(); + assert(sim[idx][pos] == real[idx][pos]); + sim[idx][pos] = *tmp; + real[idx][pos] = std::move(*tmp); + assert(real[idx].size() == old_size); + break; + } + if (existing_buffer_non_empty && command-- == 0) { + /* pop_front() */ + assert(sim[idx].front() == real[idx].front()); + size_t old_size = real[idx].size(); + sim[idx].pop_front(); + real[idx].pop_front(); + assert(real[idx].size() == old_size - 1); + break; + } + if (existing_buffer_non_empty && command-- == 0) { + /* pop_back() */ + assert(sim[idx].back() == real[idx].back()); + size_t old_size = real[idx].size(); + sim[idx].pop_back(); + real[idx].pop_back(); + assert(real[idx].size() == old_size - 1); + break; + } + } + } + + /* Fully compare the final state. */ + for (unsigned i = 0; i < sim.size(); ++i) { + // Make sure const getters work. + const VecDeque<T>& realbuf = real[i]; + const std::deque<T>& simbuf = sim[i]; + compare_fn(realbuf, simbuf); + for (unsigned j = 0; j < sim.size(); ++j) { + assert((realbuf == real[j]) == (simbuf == sim[j])); + assert(((realbuf <=> real[j]) >= 0) == (simbuf >= sim[j])); + assert(((realbuf <=> real[j]) <= 0) == (simbuf <= sim[j])); + } + // Clear out the buffers so we can check below that no objects exist anymore. + sim[i].clear(); + real[i].clear(); + } + + if constexpr (CheckNoneLeft) { + tmp = std::nullopt; + T::CheckNoneExist(); + } +} + +/** Data structure with built-in tracking of all existing objects. */ +template<size_t Size> +class TrackedObj +{ + static_assert(Size > 0); + + /* Data type for map that actually stores the object data. + * + * The key is a pointer to the TrackedObj, the value is the uint64_t it was initialized with. + * Default-constructed and moved-from objects hold an std::nullopt. + */ + using track_map_type = std::map<const TrackedObj<Size>*, std::optional<uint64_t>>; + +private: + + /** Actual map. */ + static inline track_map_type g_tracker; + + /** Iterators into the tracker map for this object. + * + * This is an array of size Size, all holding the same value, to give the object configurable + * size. The value is g_tracker.end() if this object is not fully initialized. */ + typename track_map_type::iterator m_track_entry[Size]; + + void Check() const + { + auto it = g_tracker.find(this); + for (size_t i = 0; i < Size; ++i) { + assert(m_track_entry[i] == it); + } + } + + /** Create entry for this object in g_tracker and populate m_track_entry. */ + void Register() + { + auto [it, inserted] = g_tracker.emplace(this, std::nullopt); + assert(inserted); + for (size_t i = 0; i < Size; ++i) { + m_track_entry[i] = it; + } + } + + void Deregister() + { + Check(); + assert(m_track_entry[0] != g_tracker.end()); + g_tracker.erase(m_track_entry[0]); + for (size_t i = 0; i < Size; ++i) { + m_track_entry[i] = g_tracker.end(); + } + } + + /** Get value corresponding to this object in g_tracker. */ + std::optional<uint64_t>& Deref() + { + Check(); + assert(m_track_entry[0] != g_tracker.end()); + return m_track_entry[0]->second; + } + + /** Get value corresponding to this object in g_tracker. */ + const std::optional<uint64_t>& Deref() const + { + Check(); + assert(m_track_entry[0] != g_tracker.end()); + return m_track_entry[0]->second; + } + +public: + ~TrackedObj() { Deregister(); } + TrackedObj() { Register(); } + + TrackedObj(uint64_t value) + { + Register(); + Deref() = value; + } + + TrackedObj(const TrackedObj& other) + { + Register(); + Deref() = other.Deref(); + } + + TrackedObj(TrackedObj&& other) + { + Register(); + Deref() = other.Deref(); + other.Deref() = std::nullopt; + } + + TrackedObj& operator=(const TrackedObj& other) + { + if (this == &other) return *this; + Deref() = other.Deref(); + return *this; + } + + TrackedObj& operator=(TrackedObj&& other) + { + if (this == &other) return *this; + Deref() = other.Deref(); + other.Deref() = std::nullopt; + return *this; + } + + friend bool operator==(const TrackedObj& a, const TrackedObj& b) + { + return a.Deref() == b.Deref(); + } + + friend std::strong_ordering operator<=>(const TrackedObj& a, const TrackedObj& b) + { + // Libc++ 15 & 16 do not support std::optional<T>::operator<=> yet. See + // https://reviews.llvm.org/D146392. + if (!a.Deref().has_value() || !b.Deref().has_value()) { + return a.Deref().has_value() <=> b.Deref().has_value(); + } + return *a.Deref() <=> *b.Deref(); + } + + static void CheckNoneExist() + { + assert(g_tracker.empty()); + } +}; + +} // namespace + +FUZZ_TARGET(vecdeque) +{ + // Run the test with simple uints (which satisfy all the trivial properties). + static_assert(std::is_trivially_copyable_v<uint32_t>); + static_assert(std::is_trivially_destructible_v<uint64_t>); + TestType<uint8_t, false>(buffer, 1); + TestType<uint16_t, false>(buffer, 2); + TestType<uint32_t, false>(buffer, 3); + TestType<uint64_t, false>(buffer, 4); + + // Run the test with TrackedObjs (which do not). + static_assert(!std::is_trivially_copyable_v<TrackedObj<3>>); + static_assert(!std::is_trivially_destructible_v<TrackedObj<17>>); + TestType<TrackedObj<1>, true>(buffer, 5); + TestType<TrackedObj<3>, true>(buffer, 6); + TestType<TrackedObj<17>, true>(buffer, 7); +} diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index c73b675388..8734735fd5 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -16,6 +16,8 @@ #include <boost/test/unit_test.hpp> +using util::SplitString; + BOOST_FIXTURE_TEST_SUITE(getarg_tests, BasicTestingSetup) void ResetArgs(ArgsManager& local_args, const std::string& strArg) diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index f0d2b9ed72..51f1d4c840 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -124,9 +124,9 @@ BOOST_AUTO_TEST_CASE(siphash) HashWriter ss{}; CMutableTransaction tx; - // Note these tests were originally written with tx.nVersion=1 + // Note these tests were originally written with tx.version=1 // and the test would be affected by default tx version bumps if not fixed. - tx.nVersion = 1; + tx.version = 1; ss << TX_WITH_WITNESS(tx); BOOST_CHECK_EQUAL(SipHashUint256(1, 2, ss.GetHash()), 0x79751e980c2a0a35ULL); diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index aaf4ca4977..b897a0a153 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -19,6 +19,8 @@ #include <boost/test/unit_test.hpp> +using util::ToString; + static const std::string strSecret1 = "5HxWvvfubhXpYYpS3tJkw6fq9jE9j18THftkZjHHfmFiWtmAbrj"; static const std::string strSecret2 = "5KC4ejrDjv152FGwP386VD1i2NYc5KkfSMyv1nGy1VGDxGHqVY3"; static const std::string strSecret1C = "Kwr371tjA9u2rFSMZjTNun2PXXP3WPZu2afRHTcta6KxEUdm1vEw"; @@ -201,37 +203,6 @@ BOOST_AUTO_TEST_CASE(key_signature_tests) BOOST_CHECK(found_small); } -BOOST_AUTO_TEST_CASE(key_key_negation) -{ - // create a dummy hash for signature comparison - unsigned char rnd[8]; - std::string str = "Bitcoin key verification\n"; - GetRandBytes(rnd); - uint256 hash{Hash(str, rnd)}; - - // import the static test key - CKey key = DecodeSecret(strSecret1C); - - // create a signature - std::vector<unsigned char> vch_sig; - std::vector<unsigned char> vch_sig_cmp; - key.Sign(hash, vch_sig); - - // negate the key twice - BOOST_CHECK(key.GetPubKey().data()[0] == 0x03); - key.Negate(); - // after the first negation, the signature must be different - key.Sign(hash, vch_sig_cmp); - BOOST_CHECK(vch_sig_cmp != vch_sig); - BOOST_CHECK(key.GetPubKey().data()[0] == 0x02); - key.Negate(); - // after the second negation, we should have the original key and thus the - // same signature - key.Sign(hash, vch_sig_cmp); - BOOST_CHECK(vch_sig_cmp == vch_sig); - BOOST_CHECK(key.GetPubKey().data()[0] == 0x03); -} - static CPubKey UnserializePubkey(const std::vector<uint8_t>& data) { DataStream stream{}; diff --git a/src/test/logging_tests.cpp b/src/test/logging_tests.cpp index 88e3ec94b7..cebd000df8 100644 --- a/src/test/logging_tests.cpp +++ b/src/test/logging_tests.cpp @@ -17,6 +17,9 @@ #include <boost/test/unit_test.hpp> +using util::SplitString; +using util::TrimString; + BOOST_FIXTURE_TEST_SUITE(logging_tests, BasicTestingSetup) static void ResetLogger() diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index d50af4c175..c4cf6f8a40 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -14,8 +14,10 @@ #include <test/util/txmempool.h> #include <txmempool.h> #include <uint256.h> +#include <util/check.h> #include <util/strencodings.h> #include <util/time.h> +#include <util/translation.h> #include <validation.h> #include <versionbits.h> @@ -46,7 +48,9 @@ struct MinerTestingSetup : public TestingSetup { // pointer is not accessed, when the new one should be accessed // instead. m_node.mempool.reset(); - m_node.mempool = std::make_unique<CTxMemPool>(MemPoolOptionsForTest(m_node)); + bilingual_str error; + m_node.mempool = std::make_unique<CTxMemPool>(MemPoolOptionsForTest(m_node), error); + Assert(error.empty()); return *m_node.mempool; } BlockAssembler AssemblerForTest(CTxMemPool& tx_mempool); @@ -421,7 +425,7 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: std::vector<int> prevheights; // relative height locked - tx.nVersion = 2; + tx.version = 2; tx.vin.resize(1); prevheights.resize(1); tx.vin[0].prevout.hash = txFirst[0]->GetHash(); // only 1 transaction @@ -622,7 +626,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) pblock->nVersion = VERSIONBITS_TOP_BITS; pblock->nTime = m_node.chainman->ActiveChain().Tip()->GetMedianTimePast()+1; CMutableTransaction txCoinbase(*pblock->vtx[0]); - txCoinbase.nVersion = 1; + txCoinbase.version = 1; txCoinbase.vin[0].scriptSig = CScript{} << (m_node.chainman->ActiveChain().Height() + 1) << bi.extranonce; txCoinbase.vout.resize(1); // Ignore the (optional) segwit commitment added by CreateNewBlock (as the hardcoded nonces don't account for this) txCoinbase.vout[0].scriptPubKey = CScript(); diff --git a/src/test/net_peer_connection_tests.cpp b/src/test/net_peer_connection_tests.cpp index 00bc1fdb6a..5f38ce112c 100644 --- a/src/test/net_peer_connection_tests.cpp +++ b/src/test/net_peer_connection_tests.cpp @@ -84,7 +84,7 @@ static void AddPeer(NodeId& id, std::vector<CNode*>& nodes, PeerManager& peerman BOOST_AUTO_TEST_CASE(test_addnode_getaddednodeinfo_and_connection_detection) { auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman, Params()); - auto peerman = PeerManager::make(*connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, {}); + auto peerman = PeerManager::make(*connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, *m_node.warnings, {}); NodeId id{0}; std::vector<CNode*> nodes; diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index b9dff96610..46a6a33b34 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -32,6 +32,7 @@ #include <string> using namespace std::literals; +using util::ToString; BOOST_FIXTURE_TEST_SUITE(net_tests, RegTestingSetup) diff --git a/src/test/node_warnings_tests.cpp b/src/test/node_warnings_tests.cpp new file mode 100644 index 0000000000..2bcc2c95ed --- /dev/null +++ b/src/test/node_warnings_tests.cpp @@ -0,0 +1,52 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// + +#include <node/warnings.h> +#include <util/translation.h> + +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(node_warnings_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(warnings) +{ + node::Warnings warnings; + // On pre-release builds, a warning is generated automatically + warnings.Unset(node::Warning::PRE_RELEASE_TEST_BUILD); + + // For these tests, we don't care what the exact warnings are, so + // just refer to them as warning_1 and warning_2 + const auto warning_1{node::Warning::CLOCK_OUT_OF_SYNC}; + const auto warning_2{node::Warning::FATAL_INTERNAL_ERROR}; + + // Ensure we start without any warnings + BOOST_CHECK(warnings.GetMessages().size() == 0); + // Add two warnings + BOOST_CHECK(warnings.Set(warning_1, _("warning 1"))); + BOOST_CHECK(warnings.Set(warning_2, _("warning 2"))); + // Unset the second one + BOOST_CHECK(warnings.Unset(warning_2)); + // Since it's already been unset, this should return false + BOOST_CHECK(!warnings.Unset(warning_2)); + // We should now be able to set w2 again + BOOST_CHECK(warnings.Set(warning_2, _("warning 2 - revision 1"))); + // Setting w2 again should return false since it's already set + BOOST_CHECK(!warnings.Set(warning_2, _("warning 2 - revision 2"))); + + // Verify messages are correct + const auto messages{warnings.GetMessages()}; + BOOST_CHECK(messages.size() == 2); + BOOST_CHECK(messages[0].original == "warning 1"); + BOOST_CHECK(messages[1].original == "warning 2 - revision 1"); + + // Clearing all warnings should also clear all messages + BOOST_CHECK(warnings.Unset(warning_1)); + BOOST_CHECK(warnings.Unset(warning_2)); + BOOST_CHECK(warnings.GetMessages().size() == 0); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/peerman_tests.cpp b/src/test/peerman_tests.cpp index 28866695bc..397b1d8c2d 100644 --- a/src/test/peerman_tests.cpp +++ b/src/test/peerman_tests.cpp @@ -31,7 +31,7 @@ static void mineBlock(const node::NodeContext& node, std::chrono::seconds block_ // Verifying when network-limited peer connections are desirable based on the node's proximity to the tip BOOST_AUTO_TEST_CASE(connections_desirable_service_flags) { - std::unique_ptr<PeerManager> peerman = PeerManager::make(*m_node.connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, {}); + std::unique_ptr<PeerManager> peerman = PeerManager::make(*m_node.connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, *m_node.warnings, {}); auto consensus = m_node.chainman->GetParams().GetConsensus(); // Check we start connecting to full nodes diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index 1c7d11d8a4..5089f3e8e3 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -17,6 +17,8 @@ #include <boost/test/unit_test.hpp> +using util::SplitString; + static UniValue JSON(std::string_view json) { UniValue value; @@ -552,7 +554,7 @@ BOOST_AUTO_TEST_CASE(help_example) // test different argument types const RPCArgList& args = {{"foo", "bar"}, {"b", true}, {"n", 1}}; BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", args), "> bitcoin-cli -named test foo=bar b=true n=1\n"); - BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", args), "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"foo\":\"bar\",\"b\":true,\"n\":1}}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"); + BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", args), "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"foo\":\"bar\",\"b\":true,\"n\":1}}' -H 'content-type: application/json' http://127.0.0.1:8332/\n"); // test shell escape BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", {{"foo", "b'ar"}}), "> bitcoin-cli -named test foo='b'''ar'\n"); @@ -565,7 +567,7 @@ BOOST_AUTO_TEST_CASE(help_example) obj_value.pushKV("b", false); obj_value.pushKV("n", 1); BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", {{"name", obj_value}}), "> bitcoin-cli -named test name='{\"foo\":\"bar\",\"b\":false,\"n\":1}'\n"); - BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", {{"name", obj_value}}), "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"name\":{\"foo\":\"bar\",\"b\":false,\"n\":1}}}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"); + BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", {{"name", obj_value}}), "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"name\":{\"foo\":\"bar\",\"b\":false,\"n\":1}}}' -H 'content-type: application/json' http://127.0.0.1:8332/\n"); // test array params UniValue arr_value(UniValue::VARR); @@ -573,7 +575,7 @@ BOOST_AUTO_TEST_CASE(help_example) arr_value.push_back(false); arr_value.push_back(1); BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", {{"name", arr_value}}), "> bitcoin-cli -named test name='[\"bar\",false,1]'\n"); - BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", {{"name", arr_value}}), "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"name\":[\"bar\",false,1]}}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"); + BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", {{"name", arr_value}}), "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"name\":[\"bar\",false,1]}}' -H 'content-type: application/json' http://127.0.0.1:8332/\n"); // test types don't matter for shell BOOST_CHECK_EQUAL(HelpExampleCliNamed("foo", {{"arg", true}}), HelpExampleCliNamed("foo", {{"arg", "true"}})); @@ -614,40 +616,26 @@ BOOST_AUTO_TEST_CASE(rpc_arg_helper) //! Check that `self.Arg` returns the same value as the `request.params` accessors RPCHelpMan::RPCMethodImpl check_positional = [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - BOOST_CHECK_EQUAL(self.Arg<int>(0), request.params[0].getInt<int>()); - BOOST_CHECK_EQUAL(self.Arg<std::string>(1), request.params[1].get_str()); - BOOST_CHECK_EQUAL(self.Arg<uint64_t>(2), request.params[2].isNull() ? DEFAULT_UINT64_T : request.params[2].getInt<uint64_t>()); - BOOST_CHECK_EQUAL(self.Arg<std::string>(3), request.params[3].isNull() ? DEFAULT_STRING : request.params[3].get_str()); - BOOST_CHECK_EQUAL(self.Arg<bool>(4), request.params[4].isNull() ? DEFAULT_BOOL : request.params[4].get_bool()); + BOOST_CHECK_EQUAL(self.Arg<int>("req_int"), request.params[0].getInt<int>()); + BOOST_CHECK_EQUAL(self.Arg<std::string>("req_str"), request.params[1].get_str()); + BOOST_CHECK_EQUAL(self.Arg<uint64_t>("def_uint64_t"), request.params[2].isNull() ? DEFAULT_UINT64_T : request.params[2].getInt<uint64_t>()); + BOOST_CHECK_EQUAL(self.Arg<std::string>("def_string"), request.params[3].isNull() ? DEFAULT_STRING : request.params[3].get_str()); + BOOST_CHECK_EQUAL(self.Arg<bool>("def_bool"), request.params[4].isNull() ? DEFAULT_BOOL : request.params[4].get_bool()); if (!request.params[5].isNull()) { - BOOST_CHECK_EQUAL(self.MaybeArg<double>(5).value(), request.params[5].get_real()); + BOOST_CHECK_EQUAL(self.MaybeArg<double>("opt_double").value(), request.params[5].get_real()); } else { - BOOST_CHECK(!self.MaybeArg<double>(5)); + BOOST_CHECK(!self.MaybeArg<double>("opt_double")); } if (!request.params[6].isNull()) { - BOOST_CHECK(self.MaybeArg<std::string>(6)); - BOOST_CHECK_EQUAL(*self.MaybeArg<std::string>(6), request.params[6].get_str()); + BOOST_CHECK(self.MaybeArg<std::string>("opt_string")); + BOOST_CHECK_EQUAL(*self.MaybeArg<std::string>("opt_string"), request.params[6].get_str()); } else { - BOOST_CHECK(!self.MaybeArg<std::string>(6)); + BOOST_CHECK(!self.MaybeArg<std::string>("opt_string")); } return UniValue{}; }; CheckRpc(params, UniValue{JSON(R"([5, "hello", null, null, null, null, null])")}, check_positional); CheckRpc(params, UniValue{JSON(R"([5, "hello", 4, "test", true, 1.23, "world"])")}, check_positional); - - //! Check that `self.Arg` returns the same value when using index and key - RPCHelpMan::RPCMethodImpl check_named = [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - BOOST_CHECK_EQUAL(self.Arg<int>(0), self.Arg<int>("req_int")); - BOOST_CHECK_EQUAL(self.Arg<std::string>(1), self.Arg<std::string>("req_str")); - BOOST_CHECK_EQUAL(self.Arg<uint64_t>(2), self.Arg<uint64_t>("def_uint64_t")); - BOOST_CHECK_EQUAL(self.Arg<std::string>(3), self.Arg<std::string>("def_string")); - BOOST_CHECK_EQUAL(self.Arg<bool>(4), self.Arg<bool>("def_bool")); - BOOST_CHECK(self.MaybeArg<double>(5) == self.MaybeArg<double>("opt_double")); - BOOST_CHECK(self.MaybeArg<std::string>(6) == self.MaybeArg<std::string>("opt_string")); - return UniValue{}; - }; - CheckRpc(params, UniValue{JSON(R"([5, "hello", null, null, null, null, null])")}, check_named); - CheckRpc(params, UniValue{JSON(R"([5, "hello", 4, "test", true, 1.23, "world"])")}, check_named); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/settings_tests.cpp b/src/test/settings_tests.cpp index 41190b3579..95f38fc0ce 100644 --- a/src/test/settings_tests.cpp +++ b/src/test/settings_tests.cpp @@ -21,6 +21,8 @@ #include <system_error> #include <vector> +using util::ToString; + inline bool operator==(const common::SettingsValue& a, const common::SettingsValue& b) { return a.write() == b.write(); diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index d08d5519a1..70a18835d2 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -92,7 +92,7 @@ void static RandomScript(CScript &script) { void static RandomTransaction(CMutableTransaction& tx, bool fSingle) { - tx.nVersion = int(InsecureRand32()); + tx.version = InsecureRand32(); tx.vin.clear(); tx.vout.clear(); tx.nLockTime = (InsecureRandBool()) ? InsecureRand32() : 0; diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index 2081acdf4d..aed67d5f3c 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -86,7 +86,7 @@ static ScriptError VerifyWithFlag(const CTransaction& output, const CMutableTran */ static void BuildTxs(CMutableTransaction& spendingTx, CCoinsViewCache& coins, CMutableTransaction& creationTx, const CScript& scriptPubKey, const CScript& scriptSig, const CScriptWitness& witness) { - creationTx.nVersion = 1; + creationTx.version = 1; creationTx.vin.resize(1); creationTx.vin[0].prevout.SetNull(); creationTx.vin[0].scriptSig = CScript(); @@ -94,7 +94,7 @@ static void BuildTxs(CMutableTransaction& spendingTx, CCoinsViewCache& coins, CM creationTx.vout[0].nValue = 1; creationTx.vout[0].scriptPubKey = scriptPubKey; - spendingTx.nVersion = 1; + spendingTx.version = 1; spendingTx.vin.resize(1); spendingTx.vin[0].prevout.hash = creationTx.GetHash(); spendingTx.vin[0].prevout.n = 0; diff --git a/src/test/timeoffsets_tests.cpp b/src/test/timeoffsets_tests.cpp index 008f1a9db9..5f85a5feeb 100644 --- a/src/test/timeoffsets_tests.cpp +++ b/src/test/timeoffsets_tests.cpp @@ -4,6 +4,7 @@ // #include <node/timeoffsets.h> +#include <node/warnings.h> #include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> @@ -24,7 +25,8 @@ BOOST_FIXTURE_TEST_SUITE(timeoffsets_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(timeoffsets) { - TimeOffsets offsets{}; + node::Warnings warnings{}; + TimeOffsets offsets{warnings}; BOOST_CHECK(offsets.Median() == 0s); AddMulti(offsets, {{0s, -1s, -2s, -3s}}); @@ -50,7 +52,8 @@ BOOST_AUTO_TEST_CASE(timeoffsets) static bool IsWarningRaised(const std::vector<std::chrono::seconds>& check_offsets) { - TimeOffsets offsets{}; + node::Warnings warnings{}; + TimeOffsets offsets{warnings}; AddMulti(offsets, check_offsets); return offsets.WarnIfOutOfSync(); } diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index e6cf64611e..11d5a8ae55 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -38,6 +38,9 @@ #include <univalue.h> +using util::SplitString; +using util::ToString; + typedef std::vector<unsigned char> valtype; static CFeeRate g_dust{DUST_RELAY_TX_FEE}; @@ -412,7 +415,7 @@ BOOST_AUTO_TEST_CASE(test_Get) static void CreateCreditAndSpend(const FillableSigningProvider& keystore, const CScript& outscript, CTransactionRef& output, CMutableTransaction& input, bool success = true) { CMutableTransaction outputm; - outputm.nVersion = 1; + outputm.version = 1; outputm.vin.resize(1); outputm.vin[0].prevout.SetNull(); outputm.vin[0].scriptSig = CScript(); @@ -428,7 +431,7 @@ static void CreateCreditAndSpend(const FillableSigningProvider& keystore, const assert(output->vout[0] == outputm.vout[0]); CMutableTransaction inputm; - inputm.nVersion = 1; + inputm.version = 1; inputm.vin.resize(1); inputm.vin[0].prevout.hash = output->GetHash(); inputm.vin[0].prevout.n = 0; @@ -485,7 +488,7 @@ static void ReplaceRedeemScript(CScript& script, const CScript& redeemScript) BOOST_AUTO_TEST_CASE(test_big_witness_transaction) { CMutableTransaction mtx; - mtx.nVersion = 1; + mtx.version = 1; CKey key = GenerateRandomKey(); // Need to use compressed keys in segwit or the signing will fail FillableSigningProvider keystore; @@ -779,21 +782,21 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) t.vout[0].nValue = nDustThreshold; CheckIsStandard(t); - // Disallowed nVersion - t.nVersion = -1; + // Disallowed version + t.version = std::numeric_limits<uint32_t>::max(); CheckIsNotStandard(t, "version"); - t.nVersion = 0; + t.version = 0; CheckIsNotStandard(t, "version"); - t.nVersion = TX_MAX_STANDARD_VERSION + 1; + t.version = TX_MAX_STANDARD_VERSION + 1; CheckIsNotStandard(t, "version"); - // Allowed nVersion - t.nVersion = 1; + // Allowed version + t.version = 1; CheckIsStandard(t); - t.nVersion = 2; + t.version = 2; CheckIsStandard(t); // Check dust with odd relay fee to verify rounding: @@ -904,15 +907,15 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) t.vin.clear(); t.vin.resize(2438); // size per input (empty scriptSig): 41 bytes t.vout[0].scriptPubKey = CScript() << OP_RETURN << std::vector<unsigned char>(19, 0); // output size: 30 bytes - // tx header: 12 bytes => 48 vbytes - // 2438 inputs: 2438*41 = 99958 bytes => 399832 vbytes - // 1 output: 30 bytes => 120 vbytes - // =============================== - // total: 400000 vbytes + // tx header: 12 bytes => 48 weight units + // 2438 inputs: 2438*41 = 99958 bytes => 399832 weight units + // 1 output: 30 bytes => 120 weight units + // ====================================== + // total: 400000 weight units BOOST_CHECK_EQUAL(GetTransactionWeight(CTransaction(t)), 400000); CheckIsStandard(t); - // increase output size by one byte, so we end up with 400004 vbytes + // increase output size by one byte, so we end up with 400004 weight units t.vout[0].scriptPubKey = CScript() << OP_RETURN << std::vector<unsigned char>(20, 0); // output size: 31 bytes BOOST_CHECK_EQUAL(GetTransactionWeight(CTransaction(t)), 400004); CheckIsNotStandard(t, "tx-size"); diff --git a/src/test/txpackage_tests.cpp b/src/test/txpackage_tests.cpp index 55e0c5f285..478121cc6f 100644 --- a/src/test/txpackage_tests.cpp +++ b/src/test/txpackage_tests.cpp @@ -6,6 +6,7 @@ #include <key_io.h> #include <policy/packages.h> #include <policy/policy.h> +#include <policy/rbf.h> #include <primitives/transaction.h> #include <script/script.h> #include <serialize.h> @@ -523,7 +524,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) CKey child_key = GenerateRandomKey(); CScript child_locking_script = GetScriptForDestination(WitnessV0KeyHash(child_key.GetPubKey())); CMutableTransaction mtx_child1; - mtx_child1.nVersion = 1; + mtx_child1.version = 1; mtx_child1.vin.resize(1); mtx_child1.vin[0].prevout.hash = ptx_parent->GetHash(); mtx_child1.vin[0].prevout.n = 0; @@ -651,7 +652,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) CTransactionRef ptx_grandparent2 = MakeTransactionRef(mtx_grandparent2); CMutableTransaction mtx_parent2_v1; - mtx_parent2_v1.nVersion = 1; + mtx_parent2_v1.version = 1; mtx_parent2_v1.vin.resize(1); mtx_parent2_v1.vin[0].prevout.hash = ptx_grandparent2->GetHash(); mtx_parent2_v1.vin[0].prevout.n = 0; @@ -938,4 +939,147 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); } } + +BOOST_FIXTURE_TEST_CASE(package_rbf_tests, TestChain100Setup) +{ + mineBlocks(5); + LOCK(::cs_main); + size_t expected_pool_size = m_node.mempool->size(); + CKey child_key{GenerateRandomKey()}; + CScript parent_spk = GetScriptForDestination(WitnessV0KeyHash(child_key.GetPubKey())); + CKey grandchild_key{GenerateRandomKey()}; + CScript child_spk = GetScriptForDestination(WitnessV0KeyHash(grandchild_key.GetPubKey())); + + const CAmount coinbase_value{50 * COIN}; + // Test that de-duplication works. This is not actually package rbf. + { + // 1 parent paying 200sat, 1 child paying 300sat + Package package1; + // 1 parent paying 200sat, 1 child paying 500sat + Package package2; + // Package1 and package2 have the same parent. The children conflict. + auto mtx_parent = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[0], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/parent_spk, + /*output_amount=*/coinbase_value - low_fee_amt, /*submit=*/false); + CTransactionRef tx_parent = MakeTransactionRef(mtx_parent); + package1.push_back(tx_parent); + package2.push_back(tx_parent); + + CTransactionRef tx_child_1 = MakeTransactionRef(CreateValidMempoolTransaction(tx_parent, 0, 101, child_key, child_spk, coinbase_value - low_fee_amt - 300, false)); + package1.push_back(tx_child_1); + CTransactionRef tx_child_2 = MakeTransactionRef(CreateValidMempoolTransaction(tx_parent, 0, 101, child_key, child_spk, coinbase_value - low_fee_amt - 500, false)); + package2.push_back(tx_child_2); + + LOCK(m_node.mempool->cs); + const auto submit1 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package1, /*test_accept=*/false, std::nullopt); + if (auto err_1{CheckPackageMempoolAcceptResult(package1, submit1, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_1.value()); + } + + // Check precise ResultTypes and mempool size. We know it_parent_1 and it_child_1 exist from above call + auto it_parent_1 = submit1.m_tx_results.find(tx_parent->GetWitnessHash()); + auto it_child_1 = submit1.m_tx_results.find(tx_child_1->GetWitnessHash()); + BOOST_CHECK_EQUAL(it_parent_1->second.m_result_type, MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK_EQUAL(it_child_1->second.m_result_type, MempoolAcceptResult::ResultType::VALID); + expected_pool_size += 2; + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + + const auto submit2 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package2, /*test_accept=*/false, std::nullopt); + if (auto err_2{CheckPackageMempoolAcceptResult(package2, submit2, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_2.value()); + } + + // Check precise ResultTypes and mempool size. We know it_parent_2 and it_child_2 exist from above call + auto it_parent_2 = submit2.m_tx_results.find(tx_parent->GetWitnessHash()); + auto it_child_2 = submit2.m_tx_results.find(tx_child_2->GetWitnessHash()); + BOOST_CHECK_EQUAL(it_parent_2->second.m_result_type, MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + BOOST_CHECK_EQUAL(it_child_2->second.m_result_type, MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + + // child1 has been replaced + BOOST_CHECK(!m_node.mempool->exists(GenTxid::Txid(tx_child_1->GetHash()))); + } + + // Test package rbf. + { + CTransactionRef tx_parent_1 = MakeTransactionRef(CreateValidMempoolTransaction( + m_coinbase_txns[1], /*input_vout=*/0, /*input_height=*/0, + coinbaseKey, parent_spk, coinbase_value - 200, /*submit=*/false)); + CTransactionRef tx_child_1 = MakeTransactionRef(CreateValidMempoolTransaction( + tx_parent_1, /*input_vout=*/0, /*input_height=*/101, + child_key, child_spk, coinbase_value - 400, /*submit=*/false)); + + CTransactionRef tx_parent_2 = MakeTransactionRef(CreateValidMempoolTransaction( + m_coinbase_txns[1], /*input_vout=*/0, /*input_height=*/0, + coinbaseKey, parent_spk, coinbase_value - 800, /*submit=*/false)); + CTransactionRef tx_child_2 = MakeTransactionRef(CreateValidMempoolTransaction( + tx_parent_2, /*input_vout=*/0, /*input_height=*/101, + child_key, child_spk, coinbase_value - 800 - 200, /*submit=*/false)); + + CTransactionRef tx_parent_3 = MakeTransactionRef(CreateValidMempoolTransaction( + m_coinbase_txns[1], /*input_vout=*/0, /*input_height=*/0, + coinbaseKey, parent_spk, coinbase_value - 199, /*submit=*/false)); + CTransactionRef tx_child_3 = MakeTransactionRef(CreateValidMempoolTransaction( + tx_parent_3, /*input_vout=*/0, /*input_height=*/101, + child_key, child_spk, coinbase_value - 199 - 1300, /*submit=*/false)); + + // In all packages, the parents conflict with each other + BOOST_CHECK(tx_parent_1->GetHash() != tx_parent_2->GetHash() && tx_parent_2->GetHash() != tx_parent_3->GetHash()); + + // 1 parent paying 200sat, 1 child paying 200sat. + Package package1{tx_parent_1, tx_child_1}; + // 1 parent paying 800sat, 1 child paying 200sat. + Package package2{tx_parent_2, tx_child_2}; + // 1 parent paying 199sat, 1 child paying 1300sat. + Package package3{tx_parent_3, tx_child_3}; + + const auto submit1 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package1, false, std::nullopt); + if (auto err_1{CheckPackageMempoolAcceptResult(package1, submit1, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_1.value()); + } + auto it_parent_1 = submit1.m_tx_results.find(tx_parent_1->GetWitnessHash()); + auto it_child_1 = submit1.m_tx_results.find(tx_child_1->GetWitnessHash()); + BOOST_CHECK_EQUAL(it_parent_1->second.m_result_type, MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK_EQUAL(it_child_1->second.m_result_type, MempoolAcceptResult::ResultType::VALID); + expected_pool_size += 2; + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + + // This replacement is actually not package rbf; the parent carries enough fees + // to replace the entire package on its own. + const auto submit2 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package2, false, std::nullopt); + if (auto err_2{CheckPackageMempoolAcceptResult(package2, submit2, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_2.value()); + } + auto it_parent_2 = submit2.m_tx_results.find(tx_parent_2->GetWitnessHash()); + auto it_child_2 = submit2.m_tx_results.find(tx_child_2->GetWitnessHash()); + BOOST_CHECK_EQUAL(it_parent_2->second.m_result_type, MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK_EQUAL(it_child_2->second.m_result_type, MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + + // Package RBF, in which the replacement transaction's child sponsors the fees to meet RBF feerate rules + const auto submit3 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package3, false, std::nullopt); + if (auto err_3{CheckPackageMempoolAcceptResult(package3, submit3, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_3.value()); + } + auto it_parent_3 = submit3.m_tx_results.find(tx_parent_3->GetWitnessHash()); + auto it_child_3 = submit3.m_tx_results.find(tx_child_3->GetWitnessHash()); + BOOST_CHECK_EQUAL(it_parent_3->second.m_result_type, MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK_EQUAL(it_child_3->second.m_result_type, MempoolAcceptResult::ResultType::VALID); + + // package3 was considered as a package to replace both package2 transactions + BOOST_CHECK(it_parent_3->second.m_replaced_transactions.size() == 2); + BOOST_CHECK(it_child_3->second.m_replaced_transactions.empty()); + + std::vector<Wtxid> expected_package3_wtxids({tx_parent_3->GetWitnessHash(), tx_child_3->GetWitnessHash()}); + const auto package3_total_vsize{GetVirtualTransactionSize(*tx_parent_3) + GetVirtualTransactionSize(*tx_child_3)}; + BOOST_CHECK(it_parent_3->second.m_wtxids_fee_calculations.value() == expected_package3_wtxids); + BOOST_CHECK(it_child_3->second.m_wtxids_fee_calculations.value() == expected_package3_wtxids); + BOOST_CHECK_EQUAL(it_parent_3->second.m_effective_feerate.value().GetFee(package3_total_vsize), 199 + 1300); + BOOST_CHECK_EQUAL(it_child_3->second.m_effective_feerate.value().GetFee(package3_total_vsize), 199 + 1300); + + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + } + +} BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidation_tests.cpp b/src/test/txvalidation_tests.cpp index 95583b53bf..f429f94a2f 100644 --- a/src/test/txvalidation_tests.cpp +++ b/src/test/txvalidation_tests.cpp @@ -27,7 +27,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_reject_coinbase, TestChain100Setup) CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; CMutableTransaction coinbaseTx; - coinbaseTx.nVersion = 1; + coinbaseTx.version = 1; coinbaseTx.vin.resize(1); coinbaseTx.vout.resize(1); coinbaseTx.vin[0].scriptSig = CScript() << OP_11 << OP_EQUAL; @@ -72,11 +72,11 @@ static inline std::vector<CPubKey> random_keys(size_t num_keys) { return keys; } -// Creates a placeholder tx (not valid) with 25 outputs. Specify the nVersion and the inputs. +// Creates a placeholder tx (not valid) with 25 outputs. Specify the version and the inputs. static inline CTransactionRef make_tx(const std::vector<COutPoint>& inputs, int32_t version) { CMutableTransaction mtx = CMutableTransaction{}; - mtx.nVersion = version; + mtx.version = version; mtx.vin.resize(inputs.size()); mtx.vout.resize(25); for (size_t i{0}; i < inputs.size(); ++i) { @@ -286,7 +286,7 @@ BOOST_FIXTURE_TEST_CASE(version3_tests, RegTestingSetup) script_multisig << OP_2 << OP_CHECKMULTISIG; { CMutableTransaction mtx_many_sigops = CMutableTransaction{}; - mtx_many_sigops.nVersion = 3; + mtx_many_sigops.version = TRUC_VERSION; for (const auto& outpoint : multisig_outpoints) { mtx_many_sigops.vin.emplace_back(outpoint); mtx_many_sigops.vin.back().scriptWitness.stack.emplace_back(script_multisig.begin(), script_multisig.end()); diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 790eabc7c1..78ef96a15d 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -46,7 +46,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, Dersig100Setup) spends.resize(2); for (int i = 0; i < 2; i++) { - spends[i].nVersion = 1; + spends[i].version = 1; spends[i].vin.resize(1); spends[i].vin[0].prevout.hash = m_coinbase_txns[0]->GetHash(); spends[i].vin[0].prevout.n = 0; @@ -181,7 +181,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, Dersig100Setup) // coinbase tx. CMutableTransaction spend_tx; - spend_tx.nVersion = 1; + spend_tx.version = 1; spend_tx.vin.resize(1); spend_tx.vin[0].prevout.hash = m_coinbase_txns[0]->GetHash(); spend_tx.vin[0].prevout.n = 0; @@ -243,7 +243,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, Dersig100Setup) // then test validity with P2SH. { CMutableTransaction invalid_under_p2sh_tx; - invalid_under_p2sh_tx.nVersion = 1; + invalid_under_p2sh_tx.version = 1; invalid_under_p2sh_tx.vin.resize(1); invalid_under_p2sh_tx.vin[0].prevout.hash = spend_tx.GetHash(); invalid_under_p2sh_tx.vin[0].prevout.n = 0; @@ -259,7 +259,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, Dersig100Setup) // Test CHECKLOCKTIMEVERIFY { CMutableTransaction invalid_with_cltv_tx; - invalid_with_cltv_tx.nVersion = 1; + invalid_with_cltv_tx.version = 1; invalid_with_cltv_tx.nLockTime = 100; invalid_with_cltv_tx.vin.resize(1); invalid_with_cltv_tx.vin[0].prevout.hash = spend_tx.GetHash(); @@ -288,7 +288,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, Dersig100Setup) // TEST CHECKSEQUENCEVERIFY { CMutableTransaction invalid_with_csv_tx; - invalid_with_csv_tx.nVersion = 2; + invalid_with_csv_tx.version = 2; invalid_with_csv_tx.vin.resize(1); invalid_with_csv_tx.vin[0].prevout.hash = spend_tx.GetHash(); invalid_with_csv_tx.vin[0].prevout.n = 3; @@ -319,7 +319,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, Dersig100Setup) // for the same tx with a different witness. { CMutableTransaction valid_with_witness_tx; - valid_with_witness_tx.nVersion = 1; + valid_with_witness_tx.version = 1; valid_with_witness_tx.vin.resize(1); valid_with_witness_tx.vin[0].prevout.hash = spend_tx.GetHash(); valid_with_witness_tx.vin[0].prevout.n = 1; @@ -344,7 +344,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, Dersig100Setup) // Test a transaction with multiple inputs. CMutableTransaction tx; - tx.nVersion = 1; + tx.version = 1; tx.vin.resize(2); tx.vin[0].prevout.hash = spend_tx.GetHash(); tx.vin[0].prevout.n = 0; diff --git a/src/test/uint256_tests.cpp b/src/test/uint256_tests.cpp index 5746961550..b7892a2139 100644 --- a/src/test/uint256_tests.cpp +++ b/src/test/uint256_tests.cpp @@ -267,6 +267,22 @@ BOOST_AUTO_TEST_CASE( conversion ) BOOST_AUTO_TEST_CASE( operator_with_self ) { + +/* Clang 16 and earlier detects v -= v and v /= v as self-assignments + to 0 and 1 respectively. + See: https://github.com/llvm/llvm-project/issues/42469 + and the fix in commit c5302325b2a62d77cf13dd16cd5c19141862fed0 . + + This makes some sense for arithmetic classes, but could be considered a bug + elsewhere. Disable the warning here so that the code can be tested, but the + warning should remain on as there will likely always be a better way to + express this. +*/ + +#if defined(__clang__) +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wself-assign-overloaded" +#endif arith_uint256 v = UintToArith256(uint256S("02")); v *= v; BOOST_CHECK(v == UintToArith256(uint256S("04"))); @@ -276,6 +292,9 @@ BOOST_AUTO_TEST_CASE( operator_with_self ) BOOST_CHECK(v == UintToArith256(uint256S("02"))); v -= v; BOOST_CHECK(v == UintToArith256(uint256S("0"))); +#if defined(__clang__) +# pragma clang diagnostic pop +#endif } BOOST_AUTO_TEST_CASE(parse) diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h index ff95e64b7e..03b44fc894 100644 --- a/src/test/util/chainstate.h +++ b/src/test/util/chainstate.h @@ -56,7 +56,7 @@ CreateAndActivateUTXOSnapshot( // FILE* infile{fsbridge::fopen(snapshot_path, "rb")}; AutoFile auto_infile{infile}; - node::SnapshotMetadata metadata; + node::SnapshotMetadata metadata{node.chainman->GetParams().MessageStart()}; auto_infile >> metadata; malleation(auto_infile, metadata); diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index fd07931716..79e33eacec 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -31,6 +31,7 @@ #include <node/miner.h> #include <node/peerman_args.h> #include <node/validation_cache_args.h> +#include <node/warnings.h> #include <noui.h> #include <policy/fees.h> #include <policy/fees_args.h> @@ -182,6 +183,7 @@ BasicTestingSetup::BasicTestingSetup(const ChainType chainType, const std::vecto InitLogging(*m_node.args); AppInitParameterInteraction(*m_node.args); LogInstance().StartLogging(); + m_node.warnings = std::make_unique<node::Warnings>(); m_node.kernel = std::make_unique<kernel::Context>(); m_node.ecc_context = std::make_unique<ECC_Context>(); SetupEnvironment(); @@ -227,16 +229,19 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, const std::vecto m_node.validation_signals = std::make_unique<ValidationSignals>(std::make_unique<SerialTaskRunner>(*m_node.scheduler)); m_node.fee_estimator = std::make_unique<CBlockPolicyEstimator>(FeeestPath(*m_node.args), DEFAULT_ACCEPT_STALE_FEE_ESTIMATES); - m_node.mempool = std::make_unique<CTxMemPool>(MemPoolOptionsForTest(m_node)); + bilingual_str error{}; + m_node.mempool = std::make_unique<CTxMemPool>(MemPoolOptionsForTest(m_node), error); + Assert(error.empty()); + m_node.warnings = std::make_unique<node::Warnings>(); m_cache_sizes = CalculateCacheSizes(m_args); - m_node.notifications = std::make_unique<KernelNotifications>(*Assert(m_node.shutdown), m_node.exit_status); + m_node.notifications = std::make_unique<KernelNotifications>(*Assert(m_node.shutdown), m_node.exit_status, *Assert(m_node.warnings)); const ChainstateManager::Options chainman_opts{ .chainparams = chainparams, .datadir = m_args.GetDataDirNet(), - .check_block_index = true, + .check_block_index = 1, .notifications = *m_node.notifications, .signals = m_node.validation_signals.get(), .worker_threads_num = 2, @@ -276,8 +281,8 @@ void ChainTestingSetup::LoadVerifyActivateChainstate() options.mempool = Assert(m_node.mempool.get()); options.block_tree_db_in_memory = m_block_tree_db_in_memory; options.coins_db_in_memory = m_coins_db_in_memory; - options.reindex = chainman.m_blockman.m_reindexing; - options.reindex_chainstate = m_args.GetBoolArg("-reindex-chainstate", false); + options.wipe_block_tree_db = m_args.GetBoolArg("-reindex", false); + options.wipe_chainstate_db = m_args.GetBoolArg("-reindex", false) || m_args.GetBoolArg("-reindex-chainstate", false); options.prune = chainman.m_blockman.IsPruneMode(); options.check_blocks = m_args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); options.check_level = m_args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL); @@ -320,7 +325,8 @@ TestingSetup::TestingSetup( peerman_opts.deterministic_rng = true; m_node.peerman = PeerManager::make(*m_node.connman, *m_node.addrman, m_node.banman.get(), *m_node.chainman, - *m_node.mempool, peerman_opts); + *m_node.mempool, *m_node.warnings, + peerman_opts); { CConnman::Options options; diff --git a/src/test/util/transaction_utils.cpp b/src/test/util/transaction_utils.cpp index 7e5bb30a2c..300caa577c 100644 --- a/src/test/util/transaction_utils.cpp +++ b/src/test/util/transaction_utils.cpp @@ -9,7 +9,7 @@ CMutableTransaction BuildCreditingTransaction(const CScript& scriptPubKey, int nValue) { CMutableTransaction txCredit; - txCredit.nVersion = 1; + txCredit.version = 1; txCredit.nLockTime = 0; txCredit.vin.resize(1); txCredit.vout.resize(1); @@ -25,7 +25,7 @@ CMutableTransaction BuildCreditingTransaction(const CScript& scriptPubKey, int n CMutableTransaction BuildSpendingTransaction(const CScript& scriptSig, const CScriptWitness& scriptWitness, const CTransaction& txCredit) { CMutableTransaction txSpend; - txSpend.nVersion = 1; + txSpend.version = 1; txSpend.nLockTime = 0; txSpend.vin.resize(1); txSpend.vout.resize(1); diff --git a/src/test/util/txmempool.cpp b/src/test/util/txmempool.cpp index 2657104e7d..94d50bba50 100644 --- a/src/test/util/txmempool.cpp +++ b/src/test/util/txmempool.cpp @@ -7,6 +7,7 @@ #include <chainparams.h> #include <node/context.h> #include <node/mempool_args.h> +#include <policy/rbf.h> #include <policy/v3_policy.h> #include <txmempool.h> #include <util/check.h> @@ -68,6 +69,28 @@ std::optional<std::string> CheckPackageMempoolAcceptResult(const Package& txns, return strprintf("tx %s unexpectedly failed: %s", wtxid.ToString(), atmp_result.m_state.ToString()); } + // Each subpackage is allowed MAX_REPLACEMENT_CANDIDATES replacements (only checking individually here) + if (atmp_result.m_replaced_transactions.size() > MAX_REPLACEMENT_CANDIDATES) { + return strprintf("tx %s result replaced too many transactions", + wtxid.ToString()); + } + + // Replacements can't happen for subpackages larger than 2 + if (!atmp_result.m_replaced_transactions.empty() && + atmp_result.m_wtxids_fee_calculations.has_value() && atmp_result.m_wtxids_fee_calculations.value().size() > 2) { + return strprintf("tx %s was part of a too-large package RBF subpackage", + wtxid.ToString()); + } + + if (!atmp_result.m_replaced_transactions.empty() && mempool) { + LOCK(mempool->cs); + // If replacements occurred and it used 2 transactions, this is a package RBF and should result in a cluster of size 2 + if (atmp_result.m_wtxids_fee_calculations.has_value() && atmp_result.m_wtxids_fee_calculations.value().size() == 2) { + const auto cluster = mempool->GatherClusters({tx->GetHash()}); + if (cluster.size() != 2) return strprintf("tx %s has too many ancestors or descendants for a package rbf", wtxid.ToString()); + } + } + // m_vsize and m_base_fees should exist iff the result was VALID or MEMPOOL_ENTRY const bool mempool_entry{atmp_result.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY}; if (atmp_result.m_base_fees.has_value() != (valid || mempool_entry)) { @@ -108,6 +131,11 @@ std::optional<std::string> CheckPackageMempoolAcceptResult(const Package& txns, return strprintf("wtxid %s should not be in mempool", wtxid.ToString()); } } + for (const auto& tx_ref : atmp_result.m_replaced_transactions) { + if (mempool->exists(GenTxid::Txid(tx_ref->GetHash()))) { + return strprintf("tx %s should not be in mempool as it was replaced", tx_ref->GetWitnessHash().ToString()); + } + } } } return std::nullopt; @@ -118,7 +146,7 @@ void CheckMempoolV3Invariants(const CTxMemPool& tx_pool) LOCK(tx_pool.cs); for (const auto& tx_info : tx_pool.infoAll()) { const auto& entry = *Assert(tx_pool.GetEntry(tx_info.tx->GetHash())); - if (tx_info.tx->nVersion == 3) { + if (tx_info.tx->version == TRUC_VERSION) { // Check that special maximum virtual size is respected Assert(entry.GetTxSize() <= V3_MAX_VSIZE); @@ -133,12 +161,12 @@ void CheckMempoolV3Invariants(const CTxMemPool& tx_pool) Assert(entry.GetTxSize() <= V3_CHILD_MAX_VSIZE); // All v3 transactions must only have v3 unconfirmed parents. const auto& parents = entry.GetMemPoolParentsConst(); - Assert(parents.begin()->get().GetSharedTx()->nVersion == 3); + Assert(parents.begin()->get().GetSharedTx()->version == TRUC_VERSION); } } else if (entry.GetCountWithAncestors() > 1) { // All non-v3 transactions must only have non-v3 unconfirmed parents. for (const auto& parent : entry.GetMemPoolParentsConst()) { - Assert(parent.get().GetSharedTx()->nVersion != 3); + Assert(parent.get().GetSharedTx()->version != TRUC_VERSION); } } } diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 9a2add748e..a371753adf 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -3,8 +3,10 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <clientversion.h> +#include <common/signmessage.h> // For MessageSign(), MessageVerify(), MESSAGE_MAGIC #include <hash.h> // For Hash() #include <key.h> // For CKey +#include <script/parsing.h> #include <sync.h> #include <test/util/random.h> #include <test/util/setup_common.h> @@ -12,11 +14,9 @@ #include <util/bitdeque.h> #include <util/fs.h> #include <util/fs_helpers.h> -#include <util/message.h> // For MessageSign(), MessageVerify(), MESSAGE_MAGIC #include <util/moneystr.h> #include <util/overflow.h> #include <util/readwritefile.h> -#include <util/spanparsing.h> #include <util/strencodings.h> #include <util/string.h> #include <util/time.h> @@ -45,6 +45,15 @@ #include <boost/test/unit_test.hpp> using namespace std::literals; +using util::Join; +using util::RemovePrefix; +using util::RemovePrefixView; +using util::ReplaceAll; +using util::Split; +using util::SplitString; +using util::TrimString; +using util::TrimStringView; + static const std::string STRING_WITH_EMBEDDED_NULL_CHAR{"1"s "\0" "1"s}; /* defined in logging.cpp */ @@ -1292,9 +1301,9 @@ static std::string SpanToStr(const Span<const char>& span) return std::string(span.begin(), span.end()); } -BOOST_AUTO_TEST_CASE(test_spanparsing) +BOOST_AUTO_TEST_CASE(test_script_parsing) { - using namespace spanparsing; + using namespace script; std::string input; Span<const char> sp; bool success; diff --git a/src/test/util_threadnames_tests.cpp b/src/test/util_threadnames_tests.cpp index 174052d5fa..efa0b2736b 100644 --- a/src/test/util_threadnames_tests.cpp +++ b/src/test/util_threadnames_tests.cpp @@ -13,6 +13,8 @@ #include <boost/test/unit_test.hpp> +using util::ToString; + BOOST_AUTO_TEST_SUITE(util_threadnames_tests) const std::string TEST_THREAD_NAME_BASE = "test_thread."; diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 1f6b7368a2..1641c4cd22 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -378,7 +378,7 @@ struct SnapshotTestSetup : TestChain100Setup { LOCK(::cs_main); chainman.ResetChainstates(); BOOST_CHECK_EQUAL(chainman.GetAll().size(), 0); - m_node.notifications = std::make_unique<KernelNotifications>(*Assert(m_node.shutdown), m_node.exit_status); + m_node.notifications = std::make_unique<KernelNotifications>(*Assert(m_node.shutdown), m_node.exit_status, *Assert(m_node.warnings)); const ChainstateManager::Options chainman_opts{ .chainparams = ::Params(), .datadir = chainman.m_options.datadir, diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 442c1c4d42..4f79644c8d 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -42,6 +42,10 @@ #include <event2/thread.h> #include <event2/util.h> +using util::ReplaceAll; +using util::SplitString; +using util::ToString; + /** Default control ip and port */ const std::string DEFAULT_TOR_CONTROL = "127.0.0.1:" + ToString(DEFAULT_TOR_CONTROL_PORT); /** Tor cookie size (from control-spec.txt) */ diff --git a/src/txmempool.cpp b/src/txmempool.cpp index c845944d32..10674c07ac 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -16,6 +16,7 @@ #include <policy/settings.h> #include <random.h> #include <reverse_iterator.h> +#include <tinyformat.h> #include <util/check.h> #include <util/feefrac.h> #include <util/moneystr.h> @@ -26,6 +27,7 @@ #include <util/translation.h> #include <validationinterface.h> +#include <algorithm> #include <cmath> #include <numeric> #include <optional> @@ -395,8 +397,19 @@ void CTxMemPoolEntry::UpdateAncestorState(int32_t modifySize, CAmount modifyFee, assert(int(nSigOpCostWithAncestors) >= 0); } -CTxMemPool::CTxMemPool(const Options& opts) - : m_opts{opts} +//! Clamp option values and populate the error if options are not valid. +static CTxMemPool::Options&& Flatten(CTxMemPool::Options&& opts, bilingual_str& error) +{ + opts.check_ratio = std::clamp<int>(opts.check_ratio, 0, 1'000'000); + int64_t descendant_limit_bytes = opts.limits.descendant_size_vbytes * 40; + if (opts.max_size_bytes < 0 || opts.max_size_bytes < descendant_limit_bytes) { + error = strprintf(_("-maxmempool must be at least %d MB"), std::ceil(descendant_limit_bytes / 1'000'000.0)); + } + return std::move(opts); +} + +CTxMemPool::CTxMemPool(Options opts, bilingual_str& error) + : m_opts{Flatten(std::move(opts), error)} { } diff --git a/src/txmempool.h b/src/txmempool.h index c9f6cdbfac..52f186f0ff 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -43,6 +43,8 @@ class CChain; class ValidationSignals; +struct bilingual_str; + /** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */ static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; @@ -442,7 +444,7 @@ public: * accepting transactions becomes O(N^2) where N is the number of transactions * in the pool. */ - explicit CTxMemPool(const Options& opts); + explicit CTxMemPool(Options opts, bilingual_str& error); /** * If sanity-checking is turned on, check makes sure the pool is diff --git a/src/txorphanage.cpp b/src/txorphanage.cpp index 8e76c07b07..3eaf53939d 100644 --- a/src/txorphanage.cpp +++ b/src/txorphanage.cpp @@ -119,9 +119,8 @@ void TxOrphanage::LimitOrphans(unsigned int max_orphans, FastRandomContext& rng) LOCK(m_mutex); unsigned int nEvicted = 0; - static NodeSeconds nNextSweep; auto nNow{Now<NodeSeconds>()}; - if (nNextSweep <= nNow) { + if (m_next_sweep <= nNow) { // Sweep out expired orphan pool entries: int nErased = 0; auto nMinExpTime{nNow + ORPHAN_TX_EXPIRE_TIME - ORPHAN_TX_EXPIRE_INTERVAL}; @@ -136,7 +135,7 @@ void TxOrphanage::LimitOrphans(unsigned int max_orphans, FastRandomContext& rng) } } // Sweep again 5 minutes after the next entry that expires in order to batch the linear scan. - nNextSweep = nMinExpTime + ORPHAN_TX_EXPIRE_INTERVAL; + m_next_sweep = nMinExpTime + ORPHAN_TX_EXPIRE_INTERVAL; if (nErased > 0) LogPrint(BCLog::TXPACKAGES, "Erased %d orphan tx due to expiration\n", nErased); } while (m_orphans.size() > max_orphans) diff --git a/src/txorphanage.h b/src/txorphanage.h index 47becb447d..3054396b2d 100644 --- a/src/txorphanage.h +++ b/src/txorphanage.h @@ -105,6 +105,9 @@ protected: /** Erase an orphan by wtxid */ int EraseTxNoLock(const Wtxid& wtxid) EXCLUSIVE_LOCKS_REQUIRED(m_mutex); + + /** Timestamp for the next scheduled sweep of expired orphans */ + NodeSeconds m_next_sweep GUARDED_BY(m_mutex){0s}; }; #endif // BITCOIN_TXORPHANAGE_H diff --git a/src/util/bitset.h b/src/util/bitset.h new file mode 100644 index 0000000000..6f9e808c37 --- /dev/null +++ b/src/util/bitset.h @@ -0,0 +1,527 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_BITSET_H +#define BITCOIN_UTIL_BITSET_H + +#include <util/check.h> + +#include <array> +#include <bit> +#include <cstdint> +#include <limits> +#include <type_traits> + +/* This file provides data types similar to std::bitset, but adds the following functionality: + * + * - Efficient iteration over all set bits (compatible with range-based for loops). + * - Efficient search for the first and last set bit (First() and Last()). + * - Efficient set subtraction: (a - b) implements "a and not b". + * - Efficient non-strict subset/superset testing: IsSubsetOf() and IsSupersetOf(). + * - Efficient set overlap testing: a.Overlaps(b) + * - Efficient construction of set containing 0..N-1 (S::Fill). + * - Efficient construction of a single set (S::Singleton). + * - Construction from initializer lists. + * + * Other differences: + * - BitSet<N> is a bitset that supports at least N elements, but may support more (Size() reports + * the actual number). Because the actual number is unpredictable, there are no operations that + * affect all positions (like std::bitset's operator~, flip(), or all()). + * - Various other unimplemented features. + */ + +namespace bitset_detail { + +/** Count the number of bits set in an unsigned integer type. */ +template<typename I> +unsigned inline constexpr PopCount(I v) +{ + static_assert(std::is_integral_v<I> && std::is_unsigned_v<I> && std::numeric_limits<I>::radix == 2); + constexpr auto BITS = std::numeric_limits<I>::digits; + // Algorithms from https://en.wikipedia.org/wiki/Hamming_weight#Efficient_implementation. + // These seem to be faster than std::popcount when compiling for non-SSE4 on x86_64. + if constexpr (BITS <= 32) { + v -= (v >> 1) & 0x55555555; + v = (v & 0x33333333) + ((v >> 2) & 0x33333333); + v = (v + (v >> 4)) & 0x0f0f0f0f; + if constexpr (BITS > 8) v += v >> 8; + if constexpr (BITS > 16) v += v >> 16; + return v & 0x3f; + } else { + static_assert(BITS <= 64); + v -= (v >> 1) & 0x5555555555555555; + v = (v & 0x3333333333333333) + ((v >> 2) & 0x3333333333333333); + v = (v + (v >> 4)) & 0x0f0f0f0f0f0f0f0f; + return (v * uint64_t{0x0101010101010101}) >> 56; + } +} + +/** A bitset implementation backed by a single integer of type I. */ +template<typename I> +class IntBitSet +{ + // Only binary, unsigned, integer, types allowed. + static_assert(std::is_integral_v<I> && std::is_unsigned_v<I> && std::numeric_limits<I>::radix == 2); + /** The maximum number of bits this bitset supports. */ + static constexpr unsigned MAX_SIZE = std::numeric_limits<I>::digits; + /** Integer whose bits represent this bitset. */ + I m_val; + /** Internal constructor with a given integer as contents. */ + IntBitSet(I val) noexcept : m_val{val} {} + /** Dummy type to return using end(). Only used for comparing with Iterator. */ + class IteratorEnd + { + friend class IntBitSet; + constexpr IteratorEnd() = default; + public: + constexpr IteratorEnd(const IteratorEnd&) = default; + }; + /** Iterator type returned by begin(), which efficiently iterates all 1 positions. */ + class Iterator + { + friend class IntBitSet; + I m_val; /**< The original integer's remaining bits. */ + unsigned m_pos; /** Last reported 1 position (if m_pos != 0). */ + constexpr Iterator(I val) noexcept : m_val(val), m_pos(0) + { + if (m_val != 0) m_pos = std::countr_zero(m_val); + } + public: + /** Do not allow external code to construct an Iterator. */ + Iterator() = delete; + // Copying is allowed. + constexpr Iterator(const Iterator&) noexcept = default; + constexpr Iterator& operator=(const Iterator&) noexcept = default; + /** Test whether we are done (can only compare with IteratorEnd). */ + constexpr friend bool operator==(const Iterator& a, const IteratorEnd&) noexcept + { + return a.m_val == 0; + } + /** Progress to the next 1 bit (only if != IteratorEnd). */ + constexpr Iterator& operator++() noexcept + { + Assume(m_val != 0); + m_val &= m_val - I{1U}; + if (m_val != 0) m_pos = std::countr_zero(m_val); + return *this; + } + /** Get the current bit position (only if != IteratorEnd). */ + constexpr unsigned operator*() const noexcept + { + Assume(m_val != 0); + return m_pos; + } + }; + +public: + /** Construct an all-zero bitset. */ + constexpr IntBitSet() noexcept : m_val{0} {} + /** Copy construct a bitset. */ + constexpr IntBitSet(const IntBitSet&) noexcept = default; + /** Construct from a list of values. */ + constexpr IntBitSet(std::initializer_list<unsigned> ilist) noexcept : m_val(0) + { + for (auto pos : ilist) Set(pos); + } + /** Copy assign a bitset. */ + constexpr IntBitSet& operator=(const IntBitSet&) noexcept = default; + /** Assign from a list of positions (which will be made true, all others false). */ + constexpr IntBitSet& operator=(std::initializer_list<unsigned> ilist) noexcept + { + m_val = 0; + for (auto pos : ilist) Set(pos); + return *this; + } + /** Construct a bitset with the singleton i. */ + static constexpr IntBitSet Singleton(unsigned i) noexcept + { + Assume(i < MAX_SIZE); + return IntBitSet(I(1U) << i); + } + /** Construct a bitset with bits 0..count-1 (inclusive) set to 1. */ + static constexpr IntBitSet Fill(unsigned count) noexcept + { + IntBitSet ret; + Assume(count <= MAX_SIZE); + if (count) ret.m_val = I(~I{0}) >> (MAX_SIZE - count); + return ret; + } + /** Set a bit to 1. */ + constexpr void Set(unsigned pos) noexcept + { + Assume(pos < MAX_SIZE); + m_val |= I{1U} << pos; + } + /** Set a bit to the specified value. */ + constexpr void Set(unsigned pos, bool val) noexcept + { + Assume(pos < MAX_SIZE); + m_val = (m_val & ~I(I{1U} << pos)) | (I(val) << pos); + } + /** Set a bit to 0. */ + constexpr void Reset(unsigned pos) noexcept + { + Assume(pos < MAX_SIZE); + m_val &= ~I(I{1U} << pos); + } + /** Retrieve a bit at the given position. */ + constexpr bool operator[](unsigned pos) const noexcept + { + Assume(pos < MAX_SIZE); + return (m_val >> pos) & 1U; + } + /** Compute the number of 1 bits in the bitset. */ + constexpr unsigned Count() const noexcept { return PopCount(m_val); } + /** Return the number of bits that this object holds. */ + static constexpr unsigned Size() noexcept { return MAX_SIZE; } + /** Check if all bits are 0. */ + constexpr bool None() const noexcept { return m_val == 0; } + /** Check if any bits are 1. */ + constexpr bool Any() const noexcept { return !None(); } + /** Return an object that iterates over all 1 bits (++ and * only allowed when != end()). */ + constexpr Iterator begin() const noexcept { return Iterator(m_val); } + /** Return a dummy object to compare Iterators with. */ + constexpr IteratorEnd end() const noexcept { return IteratorEnd(); } + /** Find the first element (requires Any()). */ + constexpr unsigned First() const noexcept + { + Assume(m_val != 0); + return std::countr_zero(m_val); + } + /** Find the last element (requires Any()). */ + constexpr unsigned Last() const noexcept + { + Assume(m_val != 0); + return std::bit_width(m_val) - 1; + } + /** Set this object's bits to be the binary AND between respective bits from this and a. */ + constexpr IntBitSet& operator|=(const IntBitSet& a) noexcept { m_val |= a.m_val; return *this; } + /** Set this object's bits to be the binary OR between respective bits from this and a. */ + constexpr IntBitSet& operator&=(const IntBitSet& a) noexcept { m_val &= a.m_val; return *this; } + /** Set this object's bits to be the binary AND NOT between respective bits from this and a. */ + constexpr IntBitSet& operator-=(const IntBitSet& a) noexcept { m_val &= ~a.m_val; return *this; } + /** Set this object's bits to be the binary XOR between respective bits from this as a. */ + constexpr IntBitSet& operator^=(const IntBitSet& a) noexcept { m_val ^= a.m_val; return *this; } + /** Check if the intersection between two sets is non-empty. */ + constexpr bool Overlaps(const IntBitSet& a) const noexcept { return m_val & a.m_val; } + /** Return an object with the binary AND between respective bits from a and b. */ + friend constexpr IntBitSet operator&(const IntBitSet& a, const IntBitSet& b) noexcept { return I(a.m_val & b.m_val); } + /** Return an object with the binary OR between respective bits from a and b. */ + friend constexpr IntBitSet operator|(const IntBitSet& a, const IntBitSet& b) noexcept { return I(a.m_val | b.m_val); } + /** Return an object with the binary AND NOT between respective bits from a and b. */ + friend constexpr IntBitSet operator-(const IntBitSet& a, const IntBitSet& b) noexcept { return I(a.m_val & ~b.m_val); } + /** Return an object with the binary XOR between respective bits from a and b. */ + friend constexpr IntBitSet operator^(const IntBitSet& a, const IntBitSet& b) noexcept { return I(a.m_val ^ b.m_val); } + /** Check if bitset a and bitset b are identical. */ + friend constexpr bool operator==(const IntBitSet& a, const IntBitSet& b) noexcept = default; + /** Check if bitset a is a superset of bitset b (= every 1 bit in b is also in a). */ + constexpr bool IsSupersetOf(const IntBitSet& a) const noexcept { return (a.m_val & ~m_val) == 0; } + /** Check if bitset a is a subset of bitset b (= every 1 bit in a is also in b). */ + constexpr bool IsSubsetOf(const IntBitSet& a) const noexcept { return (m_val & ~a.m_val) == 0; } + /** Swap two bitsets. */ + friend constexpr void swap(IntBitSet& a, IntBitSet& b) noexcept { std::swap(a.m_val, b.m_val); } +}; + +/** A bitset implementation backed by N integers of type I. */ +template<typename I, unsigned N> +class MultiIntBitSet +{ + // Only binary, unsigned, integer, types allowed. + static_assert(std::is_integral_v<I> && std::is_unsigned_v<I> && std::numeric_limits<I>::radix == 2); + // Cannot be empty. + static_assert(N > 0); + /** The number of bits per integer. */ + static constexpr unsigned LIMB_BITS = std::numeric_limits<I>::digits; + /** Number of elements this set type supports. */ + static constexpr unsigned MAX_SIZE = LIMB_BITS * N; + // No overflow allowed here. + static_assert(MAX_SIZE / LIMB_BITS == N); + /** Array whose member integers store the bits of the set. */ + std::array<I, N> m_val; + /** Dummy type to return using end(). Only used for comparing with Iterator. */ + class IteratorEnd + { + friend class MultiIntBitSet; + constexpr IteratorEnd() = default; + public: + constexpr IteratorEnd(const IteratorEnd&) = default; + }; + /** Iterator type returned by begin(), which efficiently iterates all 1 positions. */ + class Iterator + { + friend class MultiIntBitSet; + const std::array<I, N>* m_ptr; /**< Pointer to array to fetch bits from. */ + I m_val; /**< The remaining bits of (*m_ptr)[m_idx]. */ + unsigned m_pos; /**< The last reported position. */ + unsigned m_idx; /**< The index in *m_ptr currently being iterated over. */ + constexpr Iterator(const std::array<I, N>& ref) noexcept : m_ptr(&ref), m_idx(0) + { + do { + m_val = (*m_ptr)[m_idx]; + if (m_val) { + m_pos = std::countr_zero(m_val) + m_idx * LIMB_BITS; + break; + } + ++m_idx; + } while(m_idx < N); + } + + public: + /** Do not allow external code to construct an Iterator. */ + Iterator() = delete; + // Copying is allowed. + constexpr Iterator(const Iterator&) noexcept = default; + constexpr Iterator& operator=(const Iterator&) noexcept = default; + /** Test whether we are done (can only compare with IteratorEnd). */ + friend constexpr bool operator==(const Iterator& a, const IteratorEnd&) noexcept + { + return a.m_idx == N; + } + /** Progress to the next 1 bit (only if != IteratorEnd). */ + constexpr Iterator& operator++() noexcept + { + Assume(m_idx < N); + m_val &= m_val - I{1U}; + if (m_val == 0) { + while (true) { + ++m_idx; + if (m_idx == N) break; + m_val = (*m_ptr)[m_idx]; + if (m_val) { + m_pos = std::countr_zero(m_val) + m_idx * LIMB_BITS; + break; + } + } + } else { + m_pos = std::countr_zero(m_val) + m_idx * LIMB_BITS; + } + return *this; + } + /** Get the current bit position (only if != IteratorEnd). */ + constexpr unsigned operator*() const noexcept + { + Assume(m_idx < N); + return m_pos; + } + }; + +public: + /** Construct an all-zero bitset. */ + constexpr MultiIntBitSet() noexcept : m_val{} {} + /** Copy construct a bitset. */ + constexpr MultiIntBitSet(const MultiIntBitSet&) noexcept = default; + /** Copy assign a bitset. */ + constexpr MultiIntBitSet& operator=(const MultiIntBitSet&) noexcept = default; + /** Set a bit to 1. */ + void constexpr Set(unsigned pos) noexcept + { + Assume(pos < MAX_SIZE); + m_val[pos / LIMB_BITS] |= I{1U} << (pos % LIMB_BITS); + } + /** Set a bit to the specified value. */ + void constexpr Set(unsigned pos, bool val) noexcept + { + Assume(pos < MAX_SIZE); + m_val[pos / LIMB_BITS] = (m_val[pos / LIMB_BITS] & ~I(I{1U} << (pos % LIMB_BITS))) | + (I{val} << (pos % LIMB_BITS)); + } + /** Construct a bitset from a list of values. */ + constexpr MultiIntBitSet(std::initializer_list<unsigned> ilist) noexcept : m_val{} + { + for (auto pos : ilist) Set(pos); + } + /** Set a bitset to a list of values. */ + constexpr MultiIntBitSet& operator=(std::initializer_list<unsigned> ilist) noexcept + { + m_val.fill(0); + for (auto pos : ilist) Set(pos); + return *this; + } + /** Set a bit to 0. */ + void constexpr Reset(unsigned pos) noexcept + { + Assume(pos < MAX_SIZE); + m_val[pos / LIMB_BITS] &= ~I(I{1U} << (pos % LIMB_BITS)); + } + /** Retrieve a bit at the given position. */ + bool constexpr operator[](unsigned pos) const noexcept + { + Assume(pos < MAX_SIZE); + return (m_val[pos / LIMB_BITS] >> (pos % LIMB_BITS)) & 1U; + } + /** Construct a bitset with the singleton pos. */ + static constexpr MultiIntBitSet Singleton(unsigned pos) noexcept + { + Assume(pos < MAX_SIZE); + MultiIntBitSet ret; + ret.m_val[pos / LIMB_BITS] = I{1U} << (pos % LIMB_BITS); + return ret; + } + /** Construct a bitset with bits 0..count-1 (inclusive) set to 1. */ + static constexpr MultiIntBitSet Fill(unsigned count) noexcept + { + Assume(count <= MAX_SIZE); + MultiIntBitSet ret; + if (count) { + unsigned i = 0; + while (count > LIMB_BITS) { + ret.m_val[i++] = ~I{0}; + count -= LIMB_BITS; + } + ret.m_val[i] = I(~I{0}) >> (LIMB_BITS - count); + } + return ret; + } + /** Return the number of bits that this object holds. */ + static constexpr unsigned Size() noexcept { return MAX_SIZE; } + /** Compute the number of 1 bits in the bitset. */ + unsigned constexpr Count() const noexcept + { + unsigned ret{0}; + for (I v : m_val) ret += PopCount(v); + return ret; + } + /** Check if all bits are 0. */ + bool constexpr None() const noexcept + { + for (auto v : m_val) { + if (v != 0) return false; + } + return true; + } + /** Check if any bits are 1. */ + bool constexpr Any() const noexcept { return !None(); } + /** Return an object that iterates over all 1 bits (++ and * only allowed when != end()). */ + Iterator constexpr begin() const noexcept { return Iterator(m_val); } + /** Return a dummy object to compare Iterators with. */ + IteratorEnd constexpr end() const noexcept { return IteratorEnd(); } + /** Find the first element (requires Any()). */ + unsigned constexpr First() const noexcept + { + unsigned p = 0; + while (m_val[p] == 0) { + ++p; + Assume(p < N); + } + return std::countr_zero(m_val[p]) + p * LIMB_BITS; + } + /** Find the last element (requires Any()). */ + unsigned constexpr Last() const noexcept + { + unsigned p = N - 1; + while (m_val[p] == 0) { + Assume(p > 0); + --p; + } + return std::bit_width(m_val[p]) - 1 + p * LIMB_BITS; + } + /** Set this object's bits to be the binary OR between respective bits from this and a. */ + constexpr MultiIntBitSet& operator|=(const MultiIntBitSet& a) noexcept + { + for (unsigned i = 0; i < N; ++i) { + m_val[i] |= a.m_val[i]; + } + return *this; + } + /** Set this object's bits to be the binary AND between respective bits from this and a. */ + constexpr MultiIntBitSet& operator&=(const MultiIntBitSet& a) noexcept + { + for (unsigned i = 0; i < N; ++i) { + m_val[i] &= a.m_val[i]; + } + return *this; + } + /** Set this object's bits to be the binary AND NOT between respective bits from this and a. */ + constexpr MultiIntBitSet& operator-=(const MultiIntBitSet& a) noexcept + { + for (unsigned i = 0; i < N; ++i) { + m_val[i] &= ~a.m_val[i]; + } + return *this; + } + /** Set this object's bits to be the binary XOR between respective bits from this and a. */ + constexpr MultiIntBitSet& operator^=(const MultiIntBitSet& a) noexcept + { + for (unsigned i = 0; i < N; ++i) { + m_val[i] ^= a.m_val[i]; + } + return *this; + } + /** Check whether the intersection between two sets is non-empty. */ + constexpr bool Overlaps(const MultiIntBitSet& a) const noexcept + { + for (unsigned i = 0; i < N; ++i) { + if (m_val[i] & a.m_val[i]) return true; + } + return false; + } + /** Return an object with the binary AND between respective bits from a and b. */ + friend constexpr MultiIntBitSet operator&(const MultiIntBitSet& a, const MultiIntBitSet& b) noexcept + { + MultiIntBitSet r; + for (unsigned i = 0; i < N; ++i) { + r.m_val[i] = a.m_val[i] & b.m_val[i]; + } + return r; + } + /** Return an object with the binary OR between respective bits from a and b. */ + friend constexpr MultiIntBitSet operator|(const MultiIntBitSet& a, const MultiIntBitSet& b) noexcept + { + MultiIntBitSet r; + for (unsigned i = 0; i < N; ++i) { + r.m_val[i] = a.m_val[i] | b.m_val[i]; + } + return r; + } + /** Return an object with the binary AND NOT between respective bits from a and b. */ + friend constexpr MultiIntBitSet operator-(const MultiIntBitSet& a, const MultiIntBitSet& b) noexcept + { + MultiIntBitSet r; + for (unsigned i = 0; i < N; ++i) { + r.m_val[i] = a.m_val[i] & ~b.m_val[i]; + } + return r; + } + /** Return an object with the binary XOR between respective bits from a and b. */ + friend constexpr MultiIntBitSet operator^(const MultiIntBitSet& a, const MultiIntBitSet& b) noexcept + { + MultiIntBitSet r; + for (unsigned i = 0; i < N; ++i) { + r.m_val[i] = a.m_val[i] ^ b.m_val[i]; + } + return r; + } + /** Check if bitset a is a superset of bitset b (= every 1 bit in b is also in a). */ + constexpr bool IsSupersetOf(const MultiIntBitSet& a) const noexcept + { + for (unsigned i = 0; i < N; ++i) { + if (a.m_val[i] & ~m_val[i]) return false; + } + return true; + } + /** Check if bitset a is a subset of bitset b (= every 1 bit in a is also in b). */ + constexpr bool IsSubsetOf(const MultiIntBitSet& a) const noexcept + { + for (unsigned i = 0; i < N; ++i) { + if (m_val[i] & ~a.m_val[i]) return false; + } + return true; + } + /** Check if bitset a and bitset b are identical. */ + friend constexpr bool operator==(const MultiIntBitSet& a, const MultiIntBitSet& b) noexcept = default; + /** Swap two bitsets. */ + friend constexpr void swap(MultiIntBitSet& a, MultiIntBitSet& b) noexcept { std::swap(a.m_val, b.m_val); } +}; + +} // namespace bitset_detail + +// BitSet dispatches to IntBitSet or MultiIntBitSet as appropriate for the requested minimum number +// of bits. Use IntBitSet up to 32-bit, or up to 64-bit on 64-bit platforms; above that, use a +// MultiIntBitSet of size_t. +template<unsigned BITS> +using BitSet = std::conditional_t<(BITS <= 32), bitset_detail::IntBitSet<uint32_t>, + std::conditional_t<(BITS <= std::numeric_limits<size_t>::digits), bitset_detail::IntBitSet<size_t>, + bitset_detail::MultiIntBitSet<size_t, (BITS + std::numeric_limits<size_t>::digits - 1) / std::numeric_limits<size_t>::digits>>>; + +#endif // BITCOIN_UTIL_BITSET_H diff --git a/src/util/error.cpp b/src/util/error.cpp deleted file mode 100644 index 309877d067..0000000000 --- a/src/util/error.cpp +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) 2010-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <util/error.h> - -#include <tinyformat.h> -#include <util/translation.h> - -#include <cassert> -#include <string> - -bilingual_str TransactionErrorString(const TransactionError err) -{ - switch (err) { - case TransactionError::OK: - return Untranslated("No error"); - case TransactionError::MISSING_INPUTS: - return Untranslated("Inputs missing or spent"); - case TransactionError::ALREADY_IN_CHAIN: - return Untranslated("Transaction already in block chain"); - case TransactionError::P2P_DISABLED: - return Untranslated("Peer-to-peer functionality missing or disabled"); - case TransactionError::MEMPOOL_REJECTED: - return Untranslated("Transaction rejected by mempool"); - case TransactionError::MEMPOOL_ERROR: - return Untranslated("Mempool internal error"); - case TransactionError::INVALID_PSBT: - return Untranslated("PSBT is not well-formed"); - case TransactionError::PSBT_MISMATCH: - return Untranslated("PSBTs not compatible (different transactions)"); - case TransactionError::SIGHASH_MISMATCH: - return Untranslated("Specified sighash value does not match value stored in PSBT"); - case TransactionError::MAX_FEE_EXCEEDED: - return Untranslated("Fee exceeds maximum configured by user (e.g. -maxtxfee, maxfeerate)"); - case TransactionError::MAX_BURN_EXCEEDED: - return Untranslated("Unspendable output exceeds maximum configured by user (maxburnamount)"); - case TransactionError::EXTERNAL_SIGNER_NOT_FOUND: - return Untranslated("External signer not found"); - case TransactionError::EXTERNAL_SIGNER_FAILED: - return Untranslated("External signer failed to sign"); - case TransactionError::INVALID_PACKAGE: - return Untranslated("Transaction rejected due to invalid package"); - // no default case, so the compiler can warn about missing cases - } - assert(false); -} - -bilingual_str ResolveErrMsg(const std::string& optname, const std::string& strBind) -{ - return strprintf(_("Cannot resolve -%s address: '%s'"), optname, strBind); -} - -bilingual_str InvalidPortErrMsg(const std::string& optname, const std::string& invalid_value) -{ - return strprintf(_("Invalid port specified in %s: '%s'"), optname, invalid_value); -} - -bilingual_str AmountHighWarn(const std::string& optname) -{ - return strprintf(_("%s is set very high!"), optname); -} - -bilingual_str AmountErrMsg(const std::string& optname, const std::string& strValue) -{ - return strprintf(_("Invalid amount for -%s=<amount>: '%s'"), optname, strValue); -} diff --git a/src/util/error.h b/src/util/error.h deleted file mode 100644 index a52a8f47de..0000000000 --- a/src/util/error.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) 2010-2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_UTIL_ERROR_H -#define BITCOIN_UTIL_ERROR_H - -/** - * util/error.h is a common place for definitions of simple error types and - * string functions. Types and functions defined here should not require any - * outside dependencies. - * - * Error types defined here can be used in different parts of the - * codebase, to avoid the need to write boilerplate code catching and - * translating errors passed across wallet/node/rpc/gui code boundaries. - */ - -#include <string> - -struct bilingual_str; - -enum class TransactionError { - OK, //!< No error - MISSING_INPUTS, - ALREADY_IN_CHAIN, - P2P_DISABLED, - MEMPOOL_REJECTED, - MEMPOOL_ERROR, - INVALID_PSBT, - PSBT_MISMATCH, - SIGHASH_MISMATCH, - MAX_FEE_EXCEEDED, - MAX_BURN_EXCEEDED, - EXTERNAL_SIGNER_NOT_FOUND, - EXTERNAL_SIGNER_FAILED, - INVALID_PACKAGE, -}; - -bilingual_str TransactionErrorString(const TransactionError error); - -bilingual_str ResolveErrMsg(const std::string& optname, const std::string& strBind); - -bilingual_str InvalidPortErrMsg(const std::string& optname, const std::string& strPort); - -bilingual_str AmountHighWarn(const std::string& optname); - -bilingual_str AmountErrMsg(const std::string& optname, const std::string& strValue); - -#endif // BITCOIN_UTIL_ERROR_H diff --git a/src/util/fees.cpp b/src/util/fees.cpp deleted file mode 100644 index 8ada02ce54..0000000000 --- a/src/util/fees.cpp +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2020 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <util/fees.h> - -#include <policy/fees.h> -#include <util/strencodings.h> -#include <util/string.h> - -#include <map> -#include <string> -#include <vector> -#include <utility> - -std::string StringForFeeReason(FeeReason reason) -{ - static const std::map<FeeReason, std::string> fee_reason_strings = { - {FeeReason::NONE, "None"}, - {FeeReason::HALF_ESTIMATE, "Half Target 60% Threshold"}, - {FeeReason::FULL_ESTIMATE, "Target 85% Threshold"}, - {FeeReason::DOUBLE_ESTIMATE, "Double Target 95% Threshold"}, - {FeeReason::CONSERVATIVE, "Conservative Double Target longer horizon"}, - {FeeReason::MEMPOOL_MIN, "Mempool Min Fee"}, - {FeeReason::PAYTXFEE, "PayTxFee set"}, - {FeeReason::FALLBACK, "Fallback fee"}, - {FeeReason::REQUIRED, "Minimum Required Fee"}, - }; - auto reason_string = fee_reason_strings.find(reason); - - if (reason_string == fee_reason_strings.end()) return "Unknown"; - - return reason_string->second; -} - -const std::vector<std::pair<std::string, FeeEstimateMode>>& FeeModeMap() -{ - static const std::vector<std::pair<std::string, FeeEstimateMode>> FEE_MODES = { - {"unset", FeeEstimateMode::UNSET}, - {"economical", FeeEstimateMode::ECONOMICAL}, - {"conservative", FeeEstimateMode::CONSERVATIVE}, - }; - return FEE_MODES; -} - -std::string FeeModes(const std::string& delimiter) -{ - return Join(FeeModeMap(), delimiter, [&](const std::pair<std::string, FeeEstimateMode>& i) { return i.first; }); -} - -std::string InvalidEstimateModeErrorMessage() -{ - return "Invalid estimate_mode parameter, must be one of: \"" + FeeModes("\", \"") + "\""; -} - -bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode) -{ - auto searchkey = ToUpper(mode_string); - for (const auto& pair : FeeModeMap()) { - if (ToUpper(pair.first) == searchkey) { - fee_estimate_mode = pair.second; - return true; - } - } - return false; -} diff --git a/src/util/fees.h b/src/util/fees.h deleted file mode 100644 index 10ba1e4f85..0000000000 --- a/src/util/fees.h +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2020 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_UTIL_FEES_H -#define BITCOIN_UTIL_FEES_H - -#include <string> - -enum class FeeEstimateMode; -enum class FeeReason; - -bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode); -std::string StringForFeeReason(FeeReason reason); -std::string FeeModes(const std::string& delimiter); -std::string InvalidEstimateModeErrorMessage(); - -#endif // BITCOIN_UTIL_FEES_H diff --git a/src/util/moneystr.cpp b/src/util/moneystr.cpp index 9181329afc..1ed3b2ac96 100644 --- a/src/util/moneystr.cpp +++ b/src/util/moneystr.cpp @@ -13,6 +13,9 @@ #include <cstdint> #include <optional> +using util::ContainsNoNUL; +using util::TrimString; + std::string FormatMoney(const CAmount n) { // Note: not using straight sprintf here because we do NOT want diff --git a/src/util/spanparsing.h b/src/util/spanparsing.h deleted file mode 100644 index 765fe13aca..0000000000 --- a/src/util/spanparsing.h +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (c) 2018-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_UTIL_SPANPARSING_H -#define BITCOIN_UTIL_SPANPARSING_H - -#include <span.h> - -#include <string> -#include <string_view> -#include <vector> - -namespace spanparsing { - -/** Parse a constant. - * - * If sp's initial part matches str, sp is updated to skip that part, and true is returned. - * Otherwise sp is unmodified and false is returned. - */ -bool Const(const std::string& str, Span<const char>& sp); - -/** Parse a function call. - * - * If sp's initial part matches str + "(", and sp ends with ")", sp is updated to be the - * section between the braces, and true is returned. Otherwise sp is unmodified and false - * is returned. - */ -bool Func(const std::string& str, Span<const char>& sp); - -/** Extract the expression that sp begins with. - * - * This function will return the initial part of sp, up to (but not including) the first - * comma or closing brace, skipping ones that are surrounded by braces. So for example, - * for "foo(bar(1),2),3" the initial part "foo(bar(1),2)" will be returned. sp will be - * updated to skip the initial part that is returned. - */ -Span<const char> Expr(Span<const char>& sp); - -/** Split a string on any char found in separators, returning a vector. - * - * If sep does not occur in sp, a singleton with the entirety of sp is returned. - * - * Note that this function does not care about braces, so splitting - * "foo(bar(1),2),3) on ',' will return {"foo(bar(1)", "2)", "3)"}. - */ -template <typename T = Span<const char>> -std::vector<T> Split(const Span<const char>& sp, std::string_view separators) -{ - std::vector<T> ret; - auto it = sp.begin(); - auto start = it; - while (it != sp.end()) { - if (separators.find(*it) != std::string::npos) { - ret.emplace_back(start, it); - start = it + 1; - } - ++it; - } - ret.emplace_back(start, it); - return ret; -} - -/** Split a string on every instance of sep, returning a vector. - * - * If sep does not occur in sp, a singleton with the entirety of sp is returned. - * - * Note that this function does not care about braces, so splitting - * "foo(bar(1),2),3) on ',' will return {"foo(bar(1)", "2)", "3)"}. - */ -template <typename T = Span<const char>> -std::vector<T> Split(const Span<const char>& sp, char sep) -{ - return Split<T>(sp, std::string_view{&sep, 1}); -} - -} // namespace spanparsing - -#endif // BITCOIN_UTIL_SPANPARSING_H diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index 7b5ded2975..e030262a32 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -3,9 +3,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <span.h> #include <util/strencodings.h> +#include <crypto/hex_base.h> +#include <span.h> + #include <array> #include <cassert> #include <cstring> @@ -36,29 +38,6 @@ std::string SanitizeString(std::string_view str, int rule) return result; } -const signed char p_util_hexdigit[256] = -{ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - 0,1,2,3,4,5,6,7,8,9,-1,-1,-1,-1,-1,-1, - -1,0xa,0xb,0xc,0xd,0xe,0xf,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,0xa,0xb,0xc,0xd,0xe,0xf,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, }; - -signed char HexDigit(char c) -{ - return p_util_hexdigit[(unsigned char)c]; -} - bool IsHex(std::string_view str) { for (char c : str) { @@ -466,40 +445,6 @@ std::string Capitalize(std::string str) return str; } -namespace { - -using ByteAsHex = std::array<char, 2>; - -constexpr std::array<ByteAsHex, 256> CreateByteToHexMap() -{ - constexpr char hexmap[16] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; - - std::array<ByteAsHex, 256> byte_to_hex{}; - for (size_t i = 0; i < byte_to_hex.size(); ++i) { - byte_to_hex[i][0] = hexmap[i >> 4]; - byte_to_hex[i][1] = hexmap[i & 15]; - } - return byte_to_hex; -} - -} // namespace - -std::string HexStr(const Span<const uint8_t> s) -{ - std::string rv(s.size() * 2, '\0'); - static constexpr auto byte_to_hex = CreateByteToHexMap(); - static_assert(sizeof(byte_to_hex) == 512); - - char* it = rv.data(); - for (uint8_t v : s) { - std::memcpy(it, byte_to_hex[v].data(), 2); - it += 2; - } - - assert(it == rv.data() + rv.size()); - return rv; -} - std::optional<uint64_t> ParseByteUnits(std::string_view str, ByteUnit default_multiplier) { if (str.empty()) { diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 439678c24a..e5c2d3ddf2 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -9,6 +9,7 @@ #ifndef BITCOIN_UTIL_STRENCODINGS_H #define BITCOIN_UTIL_STRENCODINGS_H +#include <crypto/hex_base.h> // IWYU pragma: export #include <span.h> #include <util/string.h> @@ -66,7 +67,6 @@ std::vector<Byte> ParseHex(std::string_view hex_str) { return TryParseHex<Byte>(hex_str).value_or(std::vector<Byte>{}); } -signed char HexDigit(char c); /* Returns true if each character in str is a hex character, and has an even * number of hex digits.*/ bool IsHex(std::string_view str); @@ -122,7 +122,7 @@ T LocaleIndependentAtoi(std::string_view str) static_assert(std::is_integral<T>::value); T result; // Emulate atoi(...) handling of white space and leading +/-. - std::string_view s = TrimStringView(str); + std::string_view s = util::TrimStringView(str); if (!s.empty() && s[0] == '+') { if (s.length() >= 2 && s[1] == '-') { return 0; @@ -232,13 +232,6 @@ std::optional<T> ToIntegral(std::string_view str) [[nodiscard]] bool ParseUInt64(std::string_view str, uint64_t *out); /** - * Convert a span of bytes to a lower-case hexadecimal string. - */ -std::string HexStr(const Span<const uint8_t> s); -inline std::string HexStr(const Span<const char> s) { return HexStr(MakeUCharSpan(s)); } -inline std::string HexStr(const Span<const std::byte> s) { return HexStr(MakeUCharSpan(s)); } - -/** * Format a paragraph of text to a fixed width, adding spaces for * indentation to any added line. */ diff --git a/src/util/string.cpp b/src/util/string.cpp index 3d31849745..47c6b74d4f 100644 --- a/src/util/string.cpp +++ b/src/util/string.cpp @@ -7,8 +7,10 @@ #include <regex> #include <string> +namespace util { void ReplaceAll(std::string& in_out, const std::string& search, const std::string& substitute) { if (search.empty()) return; in_out = std::regex_replace(in_out, std::regex(search), substitute); } +} // namespace util diff --git a/src/util/string.h b/src/util/string.h index dab92942fb..e2e470f4ab 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_UTIL_STRING_H #define BITCOIN_UTIL_STRING_H -#include <util/spanparsing.h> +#include <span.h> #include <array> #include <cstdint> @@ -16,16 +16,54 @@ #include <string_view> // IWYU pragma: export #include <vector> +namespace util { void ReplaceAll(std::string& in_out, const std::string& search, const std::string& substitute); +/** Split a string on any char found in separators, returning a vector. + * + * If sep does not occur in sp, a singleton with the entirety of sp is returned. + * + * Note that this function does not care about braces, so splitting + * "foo(bar(1),2),3) on ',' will return {"foo(bar(1)", "2)", "3)"}. + */ +template <typename T = Span<const char>> +std::vector<T> Split(const Span<const char>& sp, std::string_view separators) +{ + std::vector<T> ret; + auto it = sp.begin(); + auto start = it; + while (it != sp.end()) { + if (separators.find(*it) != std::string::npos) { + ret.emplace_back(start, it); + start = it + 1; + } + ++it; + } + ret.emplace_back(start, it); + return ret; +} + +/** Split a string on every instance of sep, returning a vector. + * + * If sep does not occur in sp, a singleton with the entirety of sp is returned. + * + * Note that this function does not care about braces, so splitting + * "foo(bar(1),2),3) on ',' will return {"foo(bar(1)", "2)", "3)"}. + */ +template <typename T = Span<const char>> +std::vector<T> Split(const Span<const char>& sp, char sep) +{ + return Split<T>(sp, std::string_view{&sep, 1}); +} + [[nodiscard]] inline std::vector<std::string> SplitString(std::string_view str, char sep) { - return spanparsing::Split<std::string>(str, sep); + return Split<std::string>(str, sep); } [[nodiscard]] inline std::vector<std::string> SplitString(std::string_view str, std::string_view separators) { - return spanparsing::Split<std::string>(str, separators); + return Split<std::string>(str, separators); } [[nodiscard]] inline std::string_view TrimStringView(std::string_view str, std::string_view pattern = " \f\n\r\t\v") @@ -125,5 +163,6 @@ template <typename T1, size_t PREFIX_LEN> return obj.size() >= PREFIX_LEN && std::equal(std::begin(prefix), std::end(prefix), std::begin(obj)); } +} // namespace util #endif // BITCOIN_UTIL_STRING_H diff --git a/src/util/vecdeque.h b/src/util/vecdeque.h new file mode 100644 index 0000000000..a9264a5ad6 --- /dev/null +++ b/src/util/vecdeque.h @@ -0,0 +1,317 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_VECDEQUE_H +#define BITCOIN_UTIL_VECDEQUE_H + +#include <util/check.h> + +#include <cstring> +#include <memory> +#include <type_traits> + +/** Data structure largely mimicking std::deque, but using single preallocated ring buffer. + * + * - More efficient and better memory locality than std::deque. + * - Most operations ({push_,pop_,emplace_,}{front,back}(), operator[], ...) are O(1), + * unless reallocation is needed (in which case they are O(n)). + * - Supports reserve(), capacity(), shrink_to_fit() like vectors. + * - No iterator support. + * - Data is not stored in a single contiguous block, so no data(). + */ +template<typename T> +class VecDeque +{ + /** Pointer to allocated memory. Can contain constructed and uninitialized T objects. */ + T* m_buffer{nullptr}; + /** m_buffer + m_offset points to first object in queue. m_offset = 0 if m_capacity is 0; + * otherwise 0 <= m_offset < m_capacity. */ + size_t m_offset{0}; + /** Number of objects in the container. 0 <= m_size <= m_capacity. */ + size_t m_size{0}; + /** The size of m_buffer, expressed as a multiple of the size of T. */ + size_t m_capacity{0}; + + /** Returns the number of populated objects between m_offset and the end of the buffer. */ + size_t FirstPart() const noexcept { return std::min(m_capacity - m_offset, m_size); } + + void Reallocate(size_t capacity) + { + Assume(capacity >= m_size); + Assume((m_offset == 0 && m_capacity == 0) || m_offset < m_capacity); + // Allocate new buffer. + T* new_buffer = capacity ? std::allocator<T>().allocate(capacity) : nullptr; + if (capacity) { + if constexpr (std::is_trivially_copyable_v<T>) { + // When T is trivially copyable, just copy the data over from old to new buffer. + size_t first_part = FirstPart(); + if (first_part != 0) { + std::memcpy(new_buffer, m_buffer + m_offset, first_part * sizeof(T)); + } + if (first_part != m_size) { + std::memcpy(new_buffer + first_part, m_buffer, (m_size - first_part) * sizeof(T)); + } + } else { + // Otherwise move-construct in place in the new buffer, and destroy old buffer objects. + size_t old_pos = m_offset; + for (size_t new_pos = 0; new_pos < m_size; ++new_pos) { + std::construct_at(new_buffer + new_pos, std::move(*(m_buffer + old_pos))); + std::destroy_at(m_buffer + old_pos); + ++old_pos; + if (old_pos == m_capacity) old_pos = 0; + } + } + } + // Deallocate old buffer and update housekeeping. + std::allocator<T>().deallocate(m_buffer, m_capacity); + m_buffer = new_buffer; + m_offset = 0; + m_capacity = capacity; + Assume((m_offset == 0 && m_capacity == 0) || m_offset < m_capacity); + } + + /** What index in the buffer does logical entry number pos have? */ + size_t BufferIndex(size_t pos) const noexcept + { + Assume(pos < m_capacity); + // The expression below is used instead of the more obvious (pos + m_offset >= m_capacity), + // because the addition there could in theory overflow with very large deques. + if (pos >= m_capacity - m_offset) { + return (m_offset + pos) - m_capacity; + } else { + return m_offset + pos; + } + } + + /** Specialization of resize() that can only shrink. Separate so that clear() can call it + * without requiring a default T constructor. */ + void ResizeDown(size_t size) noexcept + { + Assume(size <= m_size); + if constexpr (std::is_trivially_destructible_v<T>) { + // If T is trivially destructible, we do not need to do anything but update the + // housekeeping record. Default constructor or zero-filling will be used when + // the space is reused. + m_size = size; + } else { + // If not, we need to invoke the destructor for every element separately. + while (m_size > size) { + std::destroy_at(m_buffer + BufferIndex(m_size - 1)); + --m_size; + } + } + } + +public: + VecDeque() noexcept = default; + + /** Resize the deque to be exactly size size (adding default-constructed elements if needed). */ + void resize(size_t size) + { + if (size < m_size) { + // Delegate to ResizeDown when shrinking. + ResizeDown(size); + } else if (size > m_size) { + // When growing, first see if we need to allocate more space. + if (size > m_capacity) Reallocate(size); + while (m_size < size) { + std::construct_at(m_buffer + BufferIndex(m_size)); + ++m_size; + } + } + } + + /** Resize the deque to be size 0. The capacity will remain unchanged. */ + void clear() noexcept { ResizeDown(0); } + + /** Destroy a deque. */ + ~VecDeque() + { + clear(); + Reallocate(0); + } + + /** Copy-assign a deque. */ + VecDeque& operator=(const VecDeque& other) + { + if (&other == this) [[unlikely]] return *this; + clear(); + Reallocate(other.m_size); + if constexpr (std::is_trivially_copyable_v<T>) { + size_t first_part = other.FirstPart(); + Assume(first_part > 0 || m_size == 0); + if (first_part != 0) { + std::memcpy(m_buffer, other.m_buffer + other.m_offset, first_part * sizeof(T)); + } + if (first_part != other.m_size) { + std::memcpy(m_buffer + first_part, other.m_buffer, (other.m_size - first_part) * sizeof(T)); + } + m_size = other.m_size; + } else { + while (m_size < other.m_size) { + std::construct_at(m_buffer + BufferIndex(m_size), other[m_size]); + ++m_size; + } + } + return *this; + } + + /** Swap two deques. */ + void swap(VecDeque& other) noexcept + { + std::swap(m_buffer, other.m_buffer); + std::swap(m_offset, other.m_offset); + std::swap(m_size, other.m_size); + std::swap(m_capacity, other.m_capacity); + } + + /** Non-member version of swap. */ + friend void swap(VecDeque& a, VecDeque& b) noexcept { a.swap(b); } + + /** Move-assign a deque. */ + VecDeque& operator=(VecDeque&& other) noexcept + { + swap(other); + return *this; + } + + /** Copy-construct a deque. */ + VecDeque(const VecDeque& other) { *this = other; } + /** Move-construct a deque. */ + VecDeque(VecDeque&& other) noexcept { swap(other); } + + /** Equality comparison between two deques (only compares size+contents, not capacity). */ + bool friend operator==(const VecDeque& a, const VecDeque& b) + { + if (a.m_size != b.m_size) return false; + for (size_t i = 0; i < a.m_size; ++i) { + if (a[i] != b[i]) return false; + } + return true; + } + + /** Comparison between two deques, implementing lexicographic ordering on the contents. */ + std::strong_ordering friend operator<=>(const VecDeque& a, const VecDeque& b) + { + size_t pos_a{0}, pos_b{0}; + while (pos_a < a.m_size && pos_b < b.m_size) { + auto cmp = a[pos_a++] <=> b[pos_b++]; + if (cmp != 0) return cmp; + } + return a.m_size <=> b.m_size; + } + + /** Increase the capacity to capacity. Capacity will not shrink. */ + void reserve(size_t capacity) + { + if (capacity > m_capacity) Reallocate(capacity); + } + + /** Make the capacity equal to the size. The contents does not change. */ + void shrink_to_fit() + { + if (m_capacity > m_size) Reallocate(m_size); + } + + /** Construct a new element at the end of the deque. */ + template<typename... Args> + void emplace_back(Args&&... args) + { + if (m_size == m_capacity) Reallocate((m_size + 1) * 2); + std::construct_at(m_buffer + BufferIndex(m_size), std::forward<Args>(args)...); + ++m_size; + } + + /** Move-construct a new element at the end of the deque. */ + void push_back(T&& elem) { emplace_back(std::move(elem)); } + + /** Copy-construct a new element at the end of the deque. */ + void push_back(const T& elem) { emplace_back(elem); } + + /** Construct a new element at the beginning of the deque. */ + template<typename... Args> + void emplace_front(Args&&... args) + { + if (m_size == m_capacity) Reallocate((m_size + 1) * 2); + std::construct_at(m_buffer + BufferIndex(m_capacity - 1), std::forward<Args>(args)...); + if (m_offset == 0) m_offset = m_capacity; + --m_offset; + ++m_size; + } + + /** Copy-construct a new element at the beginning of the deque. */ + void push_front(const T& elem) { emplace_front(elem); } + + /** Move-construct a new element at the beginning of the deque. */ + void push_front(T&& elem) { emplace_front(std::move(elem)); } + + /** Remove the first element of the deque. Requires !empty(). */ + void pop_front() + { + Assume(m_size); + std::destroy_at(m_buffer + m_offset); + --m_size; + ++m_offset; + if (m_offset == m_capacity) m_offset = 0; + } + + /** Remove the last element of the deque. Requires !empty(). */ + void pop_back() + { + Assume(m_size); + std::destroy_at(m_buffer + BufferIndex(m_size - 1)); + --m_size; + } + + /** Get a mutable reference to the first element of the deque. Requires !empty(). */ + T& front() noexcept + { + Assume(m_size); + return m_buffer[m_offset]; + } + + /** Get a const reference to the first element of the deque. Requires !empty(). */ + const T& front() const noexcept + { + Assume(m_size); + return m_buffer[m_offset]; + } + + /** Get a mutable reference to the last element of the deque. Requires !empty(). */ + T& back() noexcept + { + Assume(m_size); + return m_buffer[BufferIndex(m_size - 1)]; + } + + /** Get a const reference to the last element of the deque. Requires !empty(). */ + const T& back() const noexcept + { + Assume(m_size); + return m_buffer[BufferIndex(m_size - 1)]; + } + + /** Get a mutable reference to the element in the deque at the given index. Requires idx < size(). */ + T& operator[](size_t idx) noexcept + { + Assume(idx < m_size); + return m_buffer[BufferIndex(idx)]; + } + + /** Get a const reference to the element in the deque at the given index. Requires idx < size(). */ + const T& operator[](size_t idx) const noexcept + { + Assume(idx < m_size); + return m_buffer[BufferIndex(idx)]; + } + + /** Test whether the contents of this deque is empty. */ + bool empty() const noexcept { return m_size == 0; } + /** Get the number of elements in this deque. */ + size_t size() const noexcept { return m_size; } + /** Get the capacity of this deque (maximum size it can have without reallocating). */ + size_t capacity() const noexcept { return m_capacity; } +}; + +#endif // BITCOIN_UTIL_VECDEQUE_H diff --git a/src/validation.cpp b/src/validation.cpp index 2684265c39..c34d60f137 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -27,14 +27,15 @@ #include <kernel/mempool_entry.h> #include <kernel/messagestartchars.h> #include <kernel/notifications_interface.h> +#include <kernel/warning.h> #include <logging.h> #include <logging/timer.h> #include <node/blockstorage.h> #include <node/utxo_snapshot.h> -#include <policy/v3_policy.h> #include <policy/policy.h> #include <policy/rbf.h> #include <policy/settings.h> +#include <policy/v3_policy.h> #include <pow.h> #include <primitives/block.h> #include <primitives/transaction.h> @@ -57,11 +58,11 @@ #include <util/result.h> #include <util/signalinterrupt.h> #include <util/strencodings.h> +#include <util/string.h> #include <util/time.h> #include <util/trace.h> #include <util/translation.h> #include <validationinterface.h> -#include <warnings.h> #include <algorithm> #include <cassert> @@ -524,7 +525,7 @@ public: /* m_bypass_limits */ false, /* m_coins_to_uncache */ coins_to_uncache, /* m_test_accept */ false, - /* m_allow_replacement */ false, + /* m_allow_replacement */ true, /* m_allow_sibling_eviction */ false, /* m_package_submission */ true, /* m_package_feerates */ true, @@ -602,8 +603,8 @@ public: /** * Submission of a subpackage. * If subpackage size == 1, calls AcceptSingleTransaction() with adjusted ATMPArgs to avoid - * package policy restrictions like no CPFP carve out (PackageMempoolChecks) and disabled RBF - * (m_allow_replacement), and creates a PackageMempoolAcceptResult wrapping the result. + * package policy restrictions like no CPFP carve out (PackageMempoolChecks) + * and creates a PackageMempoolAcceptResult wrapping the result. * * If subpackage size > 1, calls AcceptMultipleTransactions() with the provided ATMPArgs. * @@ -666,12 +667,13 @@ private: // only tests that are fast should be done here (to avoid CPU DoS). bool PreChecks(ATMPArgs& args, Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); - // Run checks for mempool replace-by-fee. + // Run checks for mempool replace-by-fee, only used in AcceptSingleTransaction. bool ReplacementChecks(Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); // Enforce package mempool ancestor/descendant limits (distinct from individual - // ancestor/descendant limits done in PreChecks). + // ancestor/descendant limits done in PreChecks) and run Package RBF checks. bool PackageMempoolChecks(const std::vector<CTransactionRef>& txns, + std::vector<Workspace>& workspaces, int64_t total_vsize, PackageValidationState& package_state) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); @@ -831,7 +833,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // // Replaceability signaling of the original transactions may be // ignored due to node setting. - const bool allow_rbf{m_pool.m_opts.full_rbf || SignalsOptInRBF(*ptxConflicting) || ptxConflicting->nVersion == 3}; + const bool allow_rbf{m_pool.m_opts.full_rbf || SignalsOptInRBF(*ptxConflicting) || ptxConflicting->version == TRUC_VERSION}; if (!allow_rbf) { return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "txn-mempool-conflict"); } @@ -935,7 +937,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // method of ensuring the tx remains bumped. For example, the fee-bumping child could disappear // due to a replacement. // The only exception is v3 transactions. - if (!bypass_limits && ws.m_ptx->nVersion != 3 && ws.m_modified_fees < m_pool.m_opts.min_relay_feerate.GetFee(ws.m_vsize)) { + if (!bypass_limits && ws.m_ptx->version != TRUC_VERSION && ws.m_modified_fees < m_pool.m_opts.min_relay_feerate.GetFee(ws.m_vsize)) { // Even though this is a fee-related failure, this result is TX_MEMPOOL_POLICY, not // TX_RECONSIDERABLE, because it cannot be bypassed using package validation. return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "min relay fee not met", @@ -949,7 +951,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) ws.m_iters_conflicting = m_pool.GetIterSet(ws.m_conflicts); // Note that these modifications are only applicable to single transaction scenarios; - // carve-outs and package RBF are disabled for multi-transaction evaluations. + // carve-outs are disabled for multi-transaction evaluations. CTxMemPool::Limits maybe_rbf_limits = m_pool.m_opts.limits; // Calculate in-mempool ancestors, up to a limit. @@ -1017,7 +1019,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) .descendant_count = maybe_rbf_limits.descendant_count + 1, .descendant_size_vbytes = maybe_rbf_limits.descendant_size_vbytes + EXTRA_DESCENDANT_TX_SIZE_LIMIT, }; - if (ws.m_vsize > EXTRA_DESCENDANT_TX_SIZE_LIMIT || ws.m_ptx->nVersion == 3) { + if (ws.m_vsize > EXTRA_DESCENDANT_TX_SIZE_LIMIT || ws.m_ptx->version == TRUC_VERSION) { return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "too-long-mempool-chain", error_message); } if (auto ancestors_retry{m_pool.CalculateMemPoolAncestors(*entry, cpfp_carve_out_limits)}) { @@ -1088,10 +1090,9 @@ bool MemPoolAccept::ReplacementChecks(Workspace& ws) // descendant transaction of a direct conflict to pay a higher feerate than the transaction that // might replace them, under these rules. if (const auto err_string{PaysMoreThanConflicts(ws.m_iters_conflicting, newFeeRate, hash)}) { - // Even though this is a fee-related failure, this result is TX_MEMPOOL_POLICY, not - // TX_RECONSIDERABLE, because it cannot be bypassed using package validation. - // This must be changed if package RBF is enabled. - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, + // This fee-related failure is TX_RECONSIDERABLE because validating in a package may change + // the result. + return state.Invalid(TxValidationResult::TX_RECONSIDERABLE, strprintf("insufficient fee%s", ws.m_sibling_eviction ? " (including sibling eviction)" : ""), *err_string); } @@ -1116,16 +1117,15 @@ bool MemPoolAccept::ReplacementChecks(Workspace& ws) } if (const auto err_string{PaysForRBF(m_subpackage.m_conflicting_fees, ws.m_modified_fees, ws.m_vsize, m_pool.m_opts.incremental_relay_feerate, hash)}) { - // Even though this is a fee-related failure, this result is TX_MEMPOOL_POLICY, not - // TX_RECONSIDERABLE, because it cannot be bypassed using package validation. - // This must be changed if package RBF is enabled. - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, + // Result may change in a package context + return state.Invalid(TxValidationResult::TX_RECONSIDERABLE, strprintf("insufficient fee%s", ws.m_sibling_eviction ? " (including sibling eviction)" : ""), *err_string); } return true; } bool MemPoolAccept::PackageMempoolChecks(const std::vector<CTransactionRef>& txns, + std::vector<Workspace>& workspaces, const int64_t total_vsize, PackageValidationState& package_state) { @@ -1136,12 +1136,88 @@ bool MemPoolAccept::PackageMempoolChecks(const std::vector<CTransactionRef>& txn assert(std::all_of(txns.cbegin(), txns.cend(), [this](const auto& tx) { return !m_pool.exists(GenTxid::Txid(tx->GetHash()));})); + assert(txns.size() == workspaces.size()); + auto result = m_pool.CheckPackageLimits(txns, total_vsize); if (!result) { // This is a package-wide error, separate from an individual transaction error. return package_state.Invalid(PackageValidationResult::PCKG_POLICY, "package-mempool-limits", util::ErrorString(result).original); } - return true; + + // No conflicts means we're finished. Further checks are all RBF-only. + if (!m_subpackage.m_rbf) return true; + + // We're in package RBF context; replacement proposal must be size 2 + if (workspaces.size() != 2 || !Assume(IsChildWithParents(txns))) { + return package_state.Invalid(PackageValidationResult::PCKG_POLICY, "package RBF failed: package must be 1-parent-1-child"); + } + + // If the package has in-mempool ancestors, we won't consider a package RBF + // since it would result in a cluster larger than 2. + // N.B. To relax this constraint we will need to revisit how CCoinsViewMemPool::PackageAddTransaction + // is being used inside AcceptMultipleTransactions to track available inputs while processing a package. + for (const auto& ws : workspaces) { + if (!ws.m_ancestors.empty()) { + return package_state.Invalid(PackageValidationResult::PCKG_POLICY, "package RBF failed: new transaction cannot have mempool ancestors"); + } + } + + // Aggregate all conflicts into one set. + CTxMemPool::setEntries direct_conflict_iters; + for (Workspace& ws : workspaces) { + // Aggregate all conflicts into one set. + direct_conflict_iters.merge(ws.m_iters_conflicting); + } + + const auto& parent_ws = workspaces[0]; + const auto& child_ws = workspaces[1]; + + // Don't consider replacements that would cause us to remove a large number of mempool entries. + // This limit is not increased in a package RBF. Use the aggregate number of transactions. + if (const auto err_string{GetEntriesForConflicts(*child_ws.m_ptx, m_pool, direct_conflict_iters, + m_subpackage.m_all_conflicts)}) { + return package_state.Invalid(PackageValidationResult::PCKG_POLICY, + "package RBF failed: too many potential replacements", *err_string); + } + + for (CTxMemPool::txiter it : m_subpackage.m_all_conflicts) { + m_subpackage.m_conflicting_fees += it->GetModifiedFee(); + m_subpackage.m_conflicting_size += it->GetTxSize(); + } + + // Use the child as the transaction for attributing errors to. + const Txid& child_hash = child_ws.m_ptx->GetHash(); + if (const auto err_string{PaysForRBF(/*original_fees=*/m_subpackage.m_conflicting_fees, + /*replacement_fees=*/m_subpackage.m_total_modified_fees, + /*replacement_vsize=*/m_subpackage.m_total_vsize, + m_pool.m_opts.incremental_relay_feerate, child_hash)}) { + return package_state.Invalid(PackageValidationResult::PCKG_POLICY, + "package RBF failed: insufficient anti-DoS fees", *err_string); + } + + // Ensure this two transaction package is a "chunk" on its own; we don't want the child + // to be only paying anti-DoS fees + const CFeeRate parent_feerate(parent_ws.m_modified_fees, parent_ws.m_vsize); + const CFeeRate package_feerate(m_subpackage.m_total_modified_fees, m_subpackage.m_total_vsize); + if (package_feerate <= parent_feerate) { + return package_state.Invalid(PackageValidationResult::PCKG_POLICY, + "package RBF failed: package feerate is less than parent feerate", + strprintf("package feerate %s <= parent feerate is %s", package_feerate.ToString(), parent_feerate.ToString())); + } + + // Check if it's economically rational to mine this package rather than the ones it replaces. + // This takes the place of ReplacementChecks()'s PaysMoreThanConflicts() in the package RBF setting. + if (const auto err_tup{ImprovesFeerateDiagram(m_pool, direct_conflict_iters, m_subpackage.m_all_conflicts, m_subpackage.m_total_modified_fees, m_subpackage.m_total_vsize)}) { + return package_state.Invalid(PackageValidationResult::PCKG_POLICY, + "package RBF failed: " + err_tup.value().second, ""); + } + + LogPrint(BCLog::TXPACKAGES, "package RBF checks passed: parent %s (wtxid=%s), child %s (wtxid=%s)\n", + txns.front()->GetHash().ToString(), txns.front()->GetWitnessHash().ToString(), + txns.back()->GetHash().ToString(), txns.back()->GetWitnessHash().ToString()); + + + return true; } bool MemPoolAccept::PolicyScriptChecks(const ATMPArgs& args, Workspace& ws) @@ -1215,16 +1291,19 @@ bool MemPoolAccept::Finalize(const ATMPArgs& args, Workspace& ws) const bool bypass_limits = args.m_bypass_limits; std::unique_ptr<CTxMemPoolEntry>& entry = ws.m_entry; + if (!m_subpackage.m_all_conflicts.empty()) Assume(args.m_allow_replacement); // Remove conflicting transactions from the mempool for (CTxMemPool::txiter it : m_subpackage.m_all_conflicts) { - LogPrint(BCLog::MEMPOOL, "replacing tx %s (wtxid=%s) with %s (wtxid=%s) for %s additional fees, %d delta bytes\n", + LogPrint(BCLog::MEMPOOL, "replacing mempool tx %s (wtxid=%s, fees=%s, vsize=%s). New tx %s (wtxid=%s, fees=%s, vsize=%s)\n", it->GetTx().GetHash().ToString(), it->GetTx().GetWitnessHash().ToString(), + it->GetFee(), + it->GetTxSize(), hash.ToString(), tx.GetWitnessHash().ToString(), - FormatMoney(ws.m_modified_fees - m_subpackage.m_conflicting_fees), - (int)entry->GetTxSize() - (int)m_subpackage.m_conflicting_size); + entry->GetFee(), + entry->GetTxSize()); TRACE7(mempool, replaced, it->GetTx().GetHash().data(), it->GetTxSize(), @@ -1318,6 +1397,13 @@ bool MemPoolAccept::SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& std::transform(workspaces.cbegin(), workspaces.cend(), std::back_inserter(all_package_wtxids), [](const auto& ws) { return ws.m_ptx->GetWitnessHash(); }); + if (!m_subpackage.m_replaced_transactions.empty()) { + LogPrint(BCLog::MEMPOOL, "replaced %u mempool transactions with %u new one(s) for %s additional fees, %d delta bytes\n", + m_subpackage.m_replaced_transactions.size(), workspaces.size(), + m_subpackage.m_total_modified_fees - m_subpackage.m_conflicting_fees, + m_subpackage.m_total_vsize - static_cast<int>(m_subpackage.m_conflicting_size)); + } + // Add successful results. The returned results may change later if LimitMempoolSize() evicts them. for (Workspace& ws : workspaces) { const auto effective_feerate = args.m_package_feerates ? ws.m_package_feerate : @@ -1361,7 +1447,13 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef return MempoolAcceptResult::Failure(ws.m_state); } - if (m_subpackage.m_rbf && !ReplacementChecks(ws)) return MempoolAcceptResult::Failure(ws.m_state); + if (m_subpackage.m_rbf && !ReplacementChecks(ws)) { + if (ws.m_state.GetResult() == TxValidationResult::TX_RECONSIDERABLE) { + // Failed for incentives-based fee reasons. Provide the effective feerate and which tx was included. + return MempoolAcceptResult::FeeFailure(ws.m_state, CFeeRate(ws.m_modified_fees, ws.m_vsize), single_wtxid); + } + return MempoolAcceptResult::Failure(ws.m_state); + } // Perform the inexpensive checks first and avoid hashing and signature verification unless // those checks pass, to mitigate CPU exhaustion denial-of-service attacks. @@ -1393,6 +1485,13 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef m_pool.m_opts.signals->TransactionAddedToMempool(tx_info, m_pool.GetAndIncrementSequence()); } + if (!m_subpackage.m_replaced_transactions.empty()) { + LogPrint(BCLog::MEMPOOL, "replaced %u mempool transactions with 1 new transaction for %s additional fees, %d delta bytes\n", + m_subpackage.m_replaced_transactions.size(), + ws.m_modified_fees - m_subpackage.m_conflicting_fees, + ws.m_vsize - static_cast<int>(m_subpackage.m_conflicting_size)); + } + return MempoolAcceptResult::Success(std::move(m_subpackage.m_replaced_transactions), ws.m_vsize, ws.m_base_fees, effective_feerate, single_wtxid); } @@ -1434,11 +1533,14 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: } // Make the coins created by this transaction available for subsequent transactions in the - // package to spend. Since we already checked conflicts in the package and we don't allow - // replacements, we don't need to track the coins spent. Note that this logic will need to be - // updated if package replace-by-fee is allowed in the future. - assert(!args.m_allow_replacement); - assert(!m_subpackage.m_rbf); + // package to spend. If there are no conflicts within the package, no transaction can spend a coin + // needed by another transaction in the package. We also need to make sure that no package + // tx replaces (or replaces the ancestor of) the parent of another package tx. As long as we + // check these two things, we don't need to track the coins spent. + // If a package tx conflicts with a mempool tx, PackageMempoolChecks() ensures later that any package RBF attempt + // has *no* in-mempool ancestors, so we don't have to worry about subsequent transactions in + // same package spending the same in-mempool outpoints. This needs to be revisited for general + // package RBF. m_viewmempool.PackageAddTransaction(ws.m_ptx); } @@ -1479,7 +1581,7 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: // Apply package mempool ancestor/descendant limits. Skip if there is only one transaction, // because it's unnecessary. - if (txns.size() > 1 && !PackageMempoolChecks(txns, m_subpackage.m_total_vsize, package_state)) { + if (txns.size() > 1 && !PackageMempoolChecks(txns, workspaces, m_subpackage.m_total_vsize, package_state)) { return PackageMempoolAcceptResult(package_state, std::move(results)); } @@ -1921,9 +2023,11 @@ void Chainstate::CheckForkWarningConditions() if (m_chainman.m_best_invalid && m_chainman.m_best_invalid->nChainWork > m_chain.Tip()->nChainWork + (GetBlockProof(*m_chain.Tip()) * 6)) { LogPrintf("%s: Warning: Found invalid chain at least ~6 blocks longer than our best chain.\nChain state database corruption likely.\n", __func__); - SetfLargeWorkInvalidChainFound(true); + m_chainman.GetNotifications().warningSet( + kernel::Warning::LARGE_WORK_INVALID_CHAIN, + _("Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade.")); } else { - SetfLargeWorkInvalidChainFound(false); + m_chainman.GetNotifications().warningUnset(kernel::Warning::LARGE_WORK_INVALID_CHAIN); } } @@ -2700,7 +2804,7 @@ bool Chainstate::FlushStateToDisk( CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(); LOCK(m_blockman.cs_LastBlockFile); - if (m_blockman.IsPruneMode() && (m_blockman.m_check_for_pruning || nManualPruneHeight > 0) && !m_chainman.m_blockman.m_reindexing) { + if (m_blockman.IsPruneMode() && (m_blockman.m_check_for_pruning || nManualPruneHeight > 0) && m_chainman.m_blockman.m_blockfiles_indexed) { // make sure we don't prune above any of the prune locks bestblocks // pruning is height-based int last_prune{m_chain.Height()}; // last height we can prune @@ -2847,13 +2951,6 @@ void Chainstate::PruneAndFlush() } } -/** Private helper function that concatenates warning messages. */ -static void AppendWarning(bilingual_str& res, const bilingual_str& warn) -{ - if (!res.empty()) res += Untranslated(", "); - res += warn; -} - static void UpdateTipLog( const CCoinsViewCache& coins_tip, const CBlockIndex* tip, @@ -2904,7 +3001,7 @@ void Chainstate::UpdateTip(const CBlockIndex* pindexNew) g_best_block_cv.notify_all(); } - bilingual_str warning_messages; + std::vector<bilingual_str> warning_messages; if (!m_chainman.IsInitialBlockDownload()) { const CBlockIndex* pindex = pindexNew; for (int bit = 0; bit < VERSIONBITS_NUM_BITS; bit++) { @@ -2913,14 +3010,15 @@ void Chainstate::UpdateTip(const CBlockIndex* pindexNew) if (state == ThresholdState::ACTIVE || state == ThresholdState::LOCKED_IN) { const bilingual_str warning = strprintf(_("Unknown new rules activated (versionbit %i)"), bit); if (state == ThresholdState::ACTIVE) { - m_chainman.GetNotifications().warning(warning); + m_chainman.GetNotifications().warningSet(kernel::Warning::UNKNOWN_NEW_RULES_ACTIVATED, warning); } else { - AppendWarning(warning_messages, warning); + warning_messages.push_back(warning); } } } } - UpdateTipLog(coins_tip, pindexNew, params, __func__, "", warning_messages.original); + UpdateTipLog(coins_tip, pindexNew, params, __func__, "", + util::Join(warning_messages, Untranslated(", ")).original); } /** Disconnect m_chain's tip. @@ -3313,10 +3411,10 @@ bool Chainstate::ActivateBestChainStep(BlockValidationState& state, CBlockIndex* return true; } -static SynchronizationState GetSynchronizationState(bool init, bool reindexing) +static SynchronizationState GetSynchronizationState(bool init, bool blockfiles_indexed) { if (!init) return SynchronizationState::POST_INIT; - if (reindexing) return SynchronizationState::INIT_REINDEX; + if (!blockfiles_indexed) return SynchronizationState::INIT_REINDEX; return SynchronizationState::INIT_DOWNLOAD; } @@ -3338,7 +3436,7 @@ static bool NotifyHeaderTip(ChainstateManager& chainman) LOCKS_EXCLUDED(cs_main) } // Send block tip changed notifications without cs_main if (fNotify) { - chainman.GetNotifications().headerTip(GetSynchronizationState(fInitialBlockDownload, chainman.m_blockman.m_reindexing), pindexHeader->nHeight, pindexHeader->nTime, false); + chainman.GetNotifications().headerTip(GetSynchronizationState(fInitialBlockDownload, chainman.m_blockman.m_blockfiles_indexed), pindexHeader->nHeight, pindexHeader->nTime, false); } return fNotify; } @@ -3457,7 +3555,7 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< } // Always notify the UI if a new block tip was connected - if (kernel::IsInterrupted(m_chainman.GetNotifications().blockTip(GetSynchronizationState(still_in_ibd, m_chainman.m_blockman.m_reindexing), *pindexNewTip))) { + if (kernel::IsInterrupted(m_chainman.GetNotifications().blockTip(GetSynchronizationState(still_in_ibd, m_chainman.m_blockman.m_blockfiles_indexed), *pindexNewTip))) { // Just breaking and returning success for now. This could // be changed to bubble up the kernel::Interrupted value to // the caller so the caller could distinguish between @@ -3683,7 +3781,7 @@ bool Chainstate::InvalidateBlock(BlockValidationState& state, CBlockIndex* pinde // parameter indicating the source of the tip change so hooks can // distinguish user-initiated invalidateblock changes from other // changes. - (void)m_chainman.GetNotifications().blockTip(GetSynchronizationState(m_chainman.IsInitialBlockDownload(), m_chainman.m_blockman.m_reindexing), *to_mark_failed->pprev); + (void)m_chainman.GetNotifications().blockTip(GetSynchronizationState(m_chainman.IsInitialBlockDownload(), m_chainman.m_blockman.m_blockfiles_indexed), *to_mark_failed->pprev); } return true; } @@ -4322,7 +4420,7 @@ void ChainstateManager::ReportHeadersPresync(const arith_uint256& work, int64_t m_last_presync_update = now; } bool initial_download = IsInitialBlockDownload(); - GetNotifications().headerTip(GetSynchronizationState(initial_download, m_blockman.m_reindexing), height, timestamp, /*presync=*/true); + GetNotifications().headerTip(GetSynchronizationState(initial_download, m_blockman.m_blockfiles_indexed), height, timestamp, /*presync=*/true); if (initial_download) { int64_t blocks_left{(NodeClock::now() - NodeSeconds{std::chrono::seconds{timestamp}}) / GetConsensus().PowTargetSpacing()}; blocks_left = std::max<int64_t>(0, blocks_left); @@ -4849,8 +4947,7 @@ bool ChainstateManager::LoadBlockIndex() { AssertLockHeld(cs_main); // Load block index from databases - bool needs_init = m_blockman.m_reindexing; - if (!m_blockman.m_reindexing) { + if (m_blockman.m_blockfiles_indexed) { bool ret{m_blockman.LoadBlockIndexDB(SnapshotBlockhash())}; if (!ret) return false; @@ -4881,18 +4978,6 @@ bool ChainstateManager::LoadBlockIndex() if (pindex->IsValid(BLOCK_VALID_TREE) && (m_best_header == nullptr || CBlockIndexWorkComparator()(m_best_header, pindex))) m_best_header = pindex; } - - needs_init = m_blockman.m_block_index.empty(); - } - - if (needs_init) { - // Everything here is for *new* reindex/DBs. Thus, though - // LoadBlockIndexDB may have set m_reindexing if we shut down - // mid-reindex previously, we don't check m_reindexing and - // instead only check it prior to LoadBlockIndexDB to set - // needs_init. - - LogPrintf("Initializing databases...\n"); } return true; } @@ -5033,7 +5118,7 @@ void ChainstateManager::LoadExternalBlockFile( } } - if (m_blockman.IsPruneMode() && !m_blockman.m_reindexing && pblock) { + if (m_blockman.IsPruneMode() && m_blockman.m_blockfiles_indexed && pblock) { // must update the tip for pruning to work while importing with -loadblock. // this is a tradeoff to conserve disk space at the expense of time // spent updating the tip to be able to prune. @@ -5105,6 +5190,14 @@ void ChainstateManager::LoadExternalBlockFile( LogPrintf("Loaded %i blocks from external file in %dms\n", nLoaded, Ticks<std::chrono::milliseconds>(SteadyClock::now() - start)); } +bool ChainstateManager::ShouldCheckBlockIndex() const +{ + // Assert to verify Flatten() has been called. + if (!*Assert(m_options.check_block_index)) return false; + if (GetRand(*m_options.check_block_index) >= 1) return false; + return true; +} + void ChainstateManager::CheckBlockIndex() { if (!ShouldCheckBlockIndex()) { @@ -5121,19 +5214,28 @@ void ChainstateManager::CheckBlockIndex() return; } - // Build forward-pointing map of the entire block tree. + // Build forward-pointing data structure for the entire block tree. + // For performance reasons, indexes of the best header chain are stored in a vector (within CChain). + // All remaining blocks are stored in a multimap. + // The best header chain can differ from the active chain: E.g. its entries may belong to blocks that + // are not yet validated. + CChain best_hdr_chain; + assert(m_best_header); + best_hdr_chain.SetTip(*m_best_header); + std::multimap<CBlockIndex*,CBlockIndex*> forward; for (auto& [_, block_index] : m_blockman.m_block_index) { - forward.emplace(block_index.pprev, &block_index); + // Only save indexes in forward that are not part of the best header chain. + if (!best_hdr_chain.Contains(&block_index)) { + // Only genesis, which must be part of the best header chain, can have a nullptr parent. + assert(block_index.pprev); + forward.emplace(block_index.pprev, &block_index); + } } + assert(forward.size() + best_hdr_chain.Height() + 1 == m_blockman.m_block_index.size()); - assert(forward.size() == m_blockman.m_block_index.size()); - - std::pair<std::multimap<CBlockIndex*,CBlockIndex*>::iterator,std::multimap<CBlockIndex*,CBlockIndex*>::iterator> rangeGenesis = forward.equal_range(nullptr); - CBlockIndex *pindex = rangeGenesis.first->second; - rangeGenesis.first++; - assert(rangeGenesis.first == rangeGenesis.second); // There is only one index entry with parent nullptr. - + CBlockIndex* pindex = best_hdr_chain[0]; + assert(pindex); // Iterate over the entire block tree, using depth-first search. // Along the way, remember whether there are blocks on the path from genesis // block being explored which are the first to have certain properties. @@ -5345,14 +5447,21 @@ void ChainstateManager::CheckBlockIndex() // assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow // End: actual consistency checks. - // Try descending into the first subnode. + + // Try descending into the first subnode. Always process forks first and the best header chain after. snap_update_firsts(); std::pair<std::multimap<CBlockIndex*,CBlockIndex*>::iterator,std::multimap<CBlockIndex*,CBlockIndex*>::iterator> range = forward.equal_range(pindex); if (range.first != range.second) { - // A subnode was found. + // A subnode not part of the best header chain was found. pindex = range.first->second; nHeight++; continue; + } else if (best_hdr_chain.Contains(pindex)) { + // Descend further into best header chain. + nHeight++; + pindex = best_hdr_chain[nHeight]; + if (!pindex) break; // we are finished, since the best header chain is always processed last + continue; } // This is a leaf node. // Move upwards until we reach a node of which we have not yet visited the last child. @@ -5378,9 +5487,15 @@ void ChainstateManager::CheckBlockIndex() // Proceed to the next one. rangePar.first++; if (rangePar.first != rangePar.second) { - // Move to the sibling. + // Move to a sibling not part of the best header chain. pindex = rangePar.first->second; break; + } else if (pindexPar == best_hdr_chain[nHeight - 1]) { + // Move to pindex's sibling on the best-chain, if it has one. + pindex = best_hdr_chain[nHeight]; + // There will not be a next block if (and only if) parent block is the best header. + assert((pindex == nullptr) == (pindexPar == best_hdr_chain.Tip())); + break; } else { // Move up further. pindex = pindexPar; @@ -5390,8 +5505,8 @@ void ChainstateManager::CheckBlockIndex() } } - // Check that we actually traversed the entire map. - assert(nNodes == forward.size()); + // Check that we actually traversed the entire block index. + assert(nNodes == forward.size() + best_hdr_chain.Height() + 1); } std::string Chainstate::ToString() @@ -5811,8 +5926,8 @@ bool ChainstateManager::PopulateAndValidateSnapshot( bool out_of_coins{false}; try { - Txid txid; - coins_file >> txid; + std::byte left_over_byte; + coins_file >> left_over_byte; } catch (const std::ios_base::failure&) { // We expect an exception since we should be out of coins. out_of_coins = true; @@ -5950,8 +6065,8 @@ SnapshotCompletionResult ChainstateManager::MaybeCompleteSnapshotValidation() PACKAGE_NAME, snapshot_tip_height, snapshot_base_height, snapshot_base_height, PACKAGE_BUGREPORT ); - LogPrintf("[snapshot] !!! %s\n", user_error.original); - LogPrintf("[snapshot] deleting snapshot, reverting to validated chain, and stopping node\n"); + LogError("[snapshot] !!! %s\n", user_error.original); + LogError("[snapshot] deleting snapshot, reverting to validated chain, and stopping node\n"); m_active_chainstate = m_ibd_chainstate.get(); m_snapshot_chainstate->m_disabled = true; @@ -6303,7 +6418,7 @@ bool ChainstateManager::ValidatedSnapshotCleanup() fs::path p_old, fs::path p_new, const fs::filesystem_error& err) { - LogPrintf("Error renaming path (%s) -> (%s): %s\n", + LogError("[snapshot] Error renaming path (%s) -> (%s): %s\n", fs::PathToString(p_old), fs::PathToString(p_new), err.what()); GetNotifications().fatalError(strprintf(_( "Rename of '%s' -> '%s' failed. " diff --git a/src/validation.h b/src/validation.h index ea6b6cad7e..ab7891539a 100644 --- a/src/validation.h +++ b/src/validation.h @@ -938,7 +938,7 @@ public: const CChainParams& GetParams() const { return m_options.chainparams; } const Consensus::Params& GetConsensus() const { return m_options.chainparams.GetConsensus(); } - bool ShouldCheckBlockIndex() const { return *Assert(m_options.check_block_index); } + bool ShouldCheckBlockIndex() const; const arith_uint256& MinimumChainWork() const { return *Assert(m_options.minimum_chain_work); } const uint256& AssumedValidBlock() const { return *Assert(m_options.assumed_valid_block); } kernel::Notifications& GetNotifications() const { return m_options.notifications; }; diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 42615b5d42..f1706b6800 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -194,7 +194,7 @@ util::Result<SelectionResult> SelectCoinsBnB(std::vector<OutputGroup>& utxo_pool for (const size_t& i : best_selection) { result.AddInput(utxo_pool.at(i)); } - result.ComputeAndSetWaste(cost_of_change, cost_of_change, CAmount{0}); + result.RecalculateWaste(cost_of_change, cost_of_change, CAmount{0}); assert(best_waste == result.GetWaste()); return result; @@ -792,35 +792,6 @@ void OutputGroupTypeMap::Push(const OutputGroup& group, OutputType type, bool in } } -CAmount SelectionResult::GetSelectionWaste(CAmount change_cost, CAmount target, bool use_effective_value) -{ - // This function should not be called with empty inputs as that would mean the selection failed - assert(!m_selected_inputs.empty()); - - // Always consider the cost of spending an input now vs in the future. - CAmount waste = 0; - for (const auto& coin_ptr : m_selected_inputs) { - const COutput& coin = *coin_ptr; - waste += coin.GetFee() - coin.long_term_fee; - } - // Bump fee of whole selection may diverge from sum of individual bump fees - waste -= bump_fee_group_discount; - - if (change_cost) { - // Consider the cost of making change and spending it in the future - // If we aren't making change, the caller should've set change_cost to 0 - assert(change_cost > 0); - waste += change_cost; - } else { - // When we are not making change (change_cost == 0), consider the excess we are throwing away to fees - CAmount selected_effective_value = use_effective_value ? GetSelectedEffectiveValue() : GetSelectedValue(); - assert(selected_effective_value >= target); - waste += selected_effective_value - target; - } - - return waste; -} - CAmount GenerateChangeTarget(const CAmount payment_value, const CAmount change_fee, FastRandomContext& rng) { if (payment_value <= CHANGE_LOWER / 2) { @@ -839,16 +810,32 @@ void SelectionResult::SetBumpFeeDiscount(const CAmount discount) bump_fee_group_discount = discount; } - -void SelectionResult::ComputeAndSetWaste(const CAmount min_viable_change, const CAmount change_cost, const CAmount change_fee) +void SelectionResult::RecalculateWaste(const CAmount min_viable_change, const CAmount change_cost, const CAmount change_fee) { - const CAmount change = GetChange(min_viable_change, change_fee); + // This function should not be called with empty inputs as that would mean the selection failed + assert(!m_selected_inputs.empty()); + + // Always consider the cost of spending an input now vs in the future. + CAmount waste = 0; + for (const auto& coin_ptr : m_selected_inputs) { + const COutput& coin = *coin_ptr; + waste += coin.GetFee() - coin.long_term_fee; + } + // Bump fee of whole selection may diverge from sum of individual bump fees + waste -= bump_fee_group_discount; - if (change > 0) { - m_waste = GetSelectionWaste(change_cost, m_target, m_use_effective); + if (GetChange(min_viable_change, change_fee)) { + // if we have a minimum viable amount after deducting fees, account for + // cost of creating and spending change + waste += change_cost; } else { - m_waste = GetSelectionWaste(0, m_target, m_use_effective); + // When we are not making change (GetChange(…) == 0), consider the excess we are throwing away to fees + CAmount selected_effective_value = m_use_effective ? GetSelectedEffectiveValue() : GetSelectedValue(); + assert(selected_effective_value >= m_target); + waste += selected_effective_value - m_target; } + + m_waste = waste; } void SelectionResult::SetAlgoCompleted(bool algo_completed) diff --git a/src/wallet/coinselection.h b/src/wallet/coinselection.h index 80c92e1b0e..9fb000422c 100644 --- a/src/wallet/coinselection.h +++ b/src/wallet/coinselection.h @@ -350,22 +350,6 @@ private: } } - /** Compute the waste for this result given the cost of change - * and the opportunity cost of spending these inputs now vs in the future. - * If change exists, waste = change_cost + inputs * (effective_feerate - long_term_feerate) - * If no change, waste = excess + inputs * (effective_feerate - long_term_feerate) - * where excess = selected_effective_value - target - * change_cost = effective_feerate * change_output_size + long_term_feerate * change_spend_size - * - * @param[in] change_cost The cost of creating change and spending it in the future. - * Only used if there is change, in which case it must be positive. - * Must be 0 if there is no change. - * @param[in] target The amount targeted by the coin selection algorithm. - * @param[in] use_effective_value Whether to use the input's effective value (when true) or the real value (when false). - * @return The waste - */ - [[nodiscard]] CAmount GetSelectionWaste(CAmount change_cost, CAmount target, bool use_effective_value = true); - public: explicit SelectionResult(const CAmount target, SelectionAlgorithm algo) : m_target(target), m_algo(algo) {} @@ -387,8 +371,19 @@ public: /** How much individual inputs overestimated the bump fees for shared ancestries */ void SetBumpFeeDiscount(const CAmount discount); - /** Calculates and stores the waste for this selection via GetSelectionWaste */ - void ComputeAndSetWaste(const CAmount min_viable_change, const CAmount change_cost, const CAmount change_fee); + /** Calculates and stores the waste for this result given the cost of change + * and the opportunity cost of spending these inputs now vs in the future. + * If change exists, waste = change_cost + inputs * (effective_feerate - long_term_feerate) - bump_fee_group_discount + * If no change, waste = excess + inputs * (effective_feerate - long_term_feerate) - bump_fee_group_discount + * where excess = selected_effective_value - target + * change_cost = effective_feerate * change_output_size + long_term_feerate * change_spend_size + * + * @param[in] min_viable_change The minimum amount necessary to make a change output economic + * @param[in] change_cost The cost of creating a change output and spending it in the future. Only + * used if there is change, in which case it must be non-negative. + * @param[in] change_fee The fee for creating a change output + */ + void RecalculateWaste(const CAmount min_viable_change, const CAmount change_cost, const CAmount change_fee); [[nodiscard]] CAmount GetWaste() const; /** Tracks that algorithm was able to exhaustively search the entire combination space before hitting limit of tries */ diff --git a/src/wallet/external_signer_scriptpubkeyman.cpp b/src/wallet/external_signer_scriptpubkeyman.cpp index b5703fa54a..32e9941453 100644 --- a/src/wallet/external_signer_scriptpubkeyman.cpp +++ b/src/wallet/external_signer_scriptpubkeyman.cpp @@ -6,6 +6,7 @@ #include <common/args.h> #include <common/system.h> #include <external_signer.h> +#include <node/types.h> #include <wallet/external_signer_scriptpubkeyman.h> #include <iostream> @@ -17,6 +18,8 @@ #include <utility> #include <vector> +using common::PSBTError; + namespace wallet { bool ExternalSignerScriptPubKeyMan::SetupDescriptor(WalletBatch& batch, std::unique_ptr<Descriptor> desc) { @@ -76,7 +79,7 @@ util::Result<void> ExternalSignerScriptPubKeyMan::DisplayAddress(const CTxDestin } // If sign is true, transaction must previously have been filled -TransactionError ExternalSignerScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type, bool sign, bool bip32derivs, int* n_signed, bool finalize) const +std::optional<PSBTError> ExternalSignerScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type, bool sign, bool bip32derivs, int* n_signed, bool finalize) const { if (!sign) { return DescriptorScriptPubKeyMan::FillPSBT(psbt, txdata, sighash_type, false, bip32derivs, n_signed, finalize); @@ -88,14 +91,14 @@ TransactionError ExternalSignerScriptPubKeyMan::FillPSBT(PartiallySignedTransact // TODO: for multisig wallets, we should only care if all _our_ inputs are signed complete &= PSBTInputSigned(input); } - if (complete) return TransactionError::OK; + if (complete) return {}; std::string strFailReason; if(!GetExternalSigner().SignTransaction(psbt, strFailReason)) { tfm::format(std::cerr, "Failed to sign: %s\n", strFailReason); - return TransactionError::EXTERNAL_SIGNER_FAILED; + return PSBTError::EXTERNAL_SIGNER_FAILED; } if (finalize) FinalizePSBT(psbt); // This won't work in a multisig setup - return TransactionError::OK; + return {}; } } // namespace wallet diff --git a/src/wallet/external_signer_scriptpubkeyman.h b/src/wallet/external_signer_scriptpubkeyman.h index 44286456b6..10d67d2ab4 100644 --- a/src/wallet/external_signer_scriptpubkeyman.h +++ b/src/wallet/external_signer_scriptpubkeyman.h @@ -35,7 +35,7 @@ class ExternalSignerScriptPubKeyMan : public DescriptorScriptPubKeyMan */ util::Result<void> DisplayAddress(const CTxDestination& dest, const ExternalSigner& signer) const; - TransactionError FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const override; + std::optional<common::PSBTError> FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const override; }; } // namespace wallet #endif // BITCOIN_WALLET_EXTERNAL_SIGNER_SCRIPTPUBKEYMAN_H diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 6a8453965b..3184d0f3b0 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -5,6 +5,7 @@ #include <common/system.h> #include <consensus/validation.h> #include <interfaces/chain.h> +#include <node/types.h> #include <policy/fees.h> #include <policy/policy.h> #include <util/moneystr.h> @@ -92,7 +93,7 @@ static feebumper::Result CheckFeeRate(const CWallet& wallet, const CMutableTrans } CAmount new_total_fee = newFeerate.GetFee(maxTxSize) + combined_bump_fee.value(); - CFeeRate incrementalRelayFee = std::max(wallet.chain().relayIncrementalFee(), CFeeRate(WALLET_INCREMENTAL_RELAY_FEE)); + CFeeRate incrementalRelayFee = wallet.chain().relayIncrementalFee(); // Min total fee is old fee + relay fee CAmount minTotalFee = old_fee + incrementalRelayFee.GetFee(maxTxSize); @@ -343,8 +344,8 @@ bool SignTransaction(CWallet& wallet, CMutableTransaction& mtx) { // so external signers are not asked to sign more than once. bool complete; wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, false /* sign */, true /* bip32derivs */); - const TransactionError err = wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, true /* sign */, false /* bip32derivs */); - if (err != TransactionError::OK) return false; + auto err{wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, true /* sign */, false /* bip32derivs */)}; + if (err) return false; complete = FinalizeAndExtractPSBT(psbtx, mtx); return complete; } else { diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index 0c1cae7253..9fab1b2ee4 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -8,6 +8,7 @@ #include <consensus/amount.h> #include <interfaces/chain.h> #include <interfaces/handler.h> +#include <node/types.h> #include <policy/fees.h> #include <primitives/transaction.h> #include <rpc/server.h> @@ -34,6 +35,7 @@ #include <utility> #include <vector> +using common::PSBTError; using interfaces::Chain; using interfaces::FoundBlock; using interfaces::Handler; @@ -389,7 +391,7 @@ public: } return {}; } - TransactionError fillPSBT(int sighash_type, + std::optional<PSBTError> fillPSBT(int sighash_type, bool sign, bool bip32derivs, size_t* n_signed, diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 8b78a670e4..fe35f6b223 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -21,6 +21,8 @@ #include <system_error> +using util::Join; + namespace wallet { bool VerifyWallets(WalletContext& context) { diff --git a/src/wallet/migrate.cpp b/src/wallet/migrate.cpp index 09254a76ad..d7d8577374 100644 --- a/src/wallet/migrate.cpp +++ b/src/wallet/migrate.cpp @@ -551,7 +551,7 @@ void BerkeleyRODatabase::Open() // } // Check the last page number - uint32_t expected_last_page = (size / page_size) - 1; + uint32_t expected_last_page{uint32_t((size / page_size) - 1)}; if (outer_meta.last_page != expected_last_page) { throw std::runtime_error("Last page number could not fit in file"); } diff --git a/src/wallet/rpc/addresses.cpp b/src/wallet/rpc/addresses.cpp index 17bb6320a1..0c2ad06eea 100644 --- a/src/wallet/rpc/addresses.cpp +++ b/src/wallet/rpc/addresses.cpp @@ -287,9 +287,30 @@ RPCHelpMan addmultisigaddress() output_type = parsed.value(); } - // Construct using pay-to-script-hash: + // Construct multisig scripts + FlatSigningProvider provider; CScript inner; - CTxDestination dest = AddAndGetMultisigDestination(required, pubkeys, output_type, spk_man, inner); + CTxDestination dest = AddAndGetMultisigDestination(required, pubkeys, output_type, provider, inner); + + // Import scripts into the wallet + for (const auto& [id, script] : provider.scripts) { + // Due to a bug in the legacy wallet, the p2sh maximum script size limit is also imposed on 'p2sh-segwit' and 'bech32' redeem scripts. + // Even when redeem scripts over MAX_SCRIPT_ELEMENT_SIZE bytes are valid for segwit output types, we don't want to + // enable it because: + // 1) It introduces a compatibility-breaking change requiring downgrade protection; older wallets would be unable to interact with these "new" legacy wallets. + // 2) Considering the ongoing deprecation of the legacy spkm, this issue adds another good reason to transition towards descriptors. + if (script.size() > MAX_SCRIPT_ELEMENT_SIZE) throw JSONRPCError(RPC_WALLET_ERROR, "Unsupported multisig script size for legacy wallet. Upgrade to descriptors to overcome this limitation for p2sh-segwit or bech32 scripts"); + + if (!spk_man.AddCScript(script)) { + if (CScript inner_script; spk_man.GetCScript(CScriptID(script), inner_script)) { + CHECK_NONFATAL(inner_script == script); // Nothing to add, script already contained by the wallet + continue; + } + throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Error importing script into the wallet")); + } + } + + // Store destination in the addressbook pwallet->SetAddressBook(dest, label, AddressPurpose::SEND); // Make the descriptor diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index a76ae7196c..8cddb8b099 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -34,6 +34,7 @@ using interfaces::FoundBlock; +using util::SplitString; namespace wallet { std::string static EncodeDumpString(const std::string &str) { diff --git a/src/wallet/rpc/signmessage.cpp b/src/wallet/rpc/signmessage.cpp index c9fb693482..edf93ecab7 100644 --- a/src/wallet/rpc/signmessage.cpp +++ b/src/wallet/rpc/signmessage.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/signmessage.h> #include <key_io.h> #include <rpc/util.h> -#include <util/message.h> #include <wallet/rpc/util.h> #include <wallet/wallet.h> diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index 86c5492e01..ac2a4826f0 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -2,14 +2,15 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/messages.h> #include <consensus/validation.h> #include <core_io.h> #include <key_io.h> +#include <node/types.h> #include <policy/policy.h> #include <rpc/rawtransaction_util.h> #include <rpc/util.h> #include <script/script.h> -#include <util/fees.h> #include <util/rbf.h> #include <util/translation.h> #include <util/vector.h> @@ -22,6 +23,12 @@ #include <univalue.h> +using common::FeeModeFromString; +using common::FeeModes; +using common::InvalidEstimateModeErrorMessage; +using common::StringForFeeReason; +using common::TransactionErrorString; +using node::TransactionError; namespace wallet { std::vector<CRecipient> CreateRecipients(const std::vector<std::pair<CTxDestination, CAmount>>& outputs, const std::set<int>& subtract_fee_outputs) @@ -97,9 +104,9 @@ static UniValue FinishTransaction(const std::shared_ptr<CWallet> pwallet, const // so external signers are not asked to sign more than once. bool complete; pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, /*sign=*/false, /*bip32derivs=*/true); - const TransactionError err{pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, /*sign=*/true, /*bip32derivs=*/false)}; - if (err != TransactionError::OK) { - throw JSONRPCTransactionError(err); + const auto err{pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, /*sign=*/true, /*bip32derivs=*/false)}; + if (err) { + throw JSONRPCPSBTError(*err); } CMutableTransaction mtx; @@ -1153,8 +1160,8 @@ static RPCHelpMan bumpfee_helper(std::string method_name) } else { PartiallySignedTransaction psbtx(mtx); bool complete = false; - const TransactionError err = pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, /*sign=*/false, /*bip32derivs=*/true); - CHECK_NONFATAL(err == TransactionError::OK); + const auto err{pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, /*sign=*/false, /*bip32derivs=*/true)}; + CHECK_NONFATAL(!err); CHECK_NONFATAL(!complete); DataStream ssTx{}; ssTx << psbtx; @@ -1295,7 +1302,7 @@ RPCHelpMan sendall() { return RPCHelpMan{"sendall", "EXPERIMENTAL warning: this call may be changed in future releases.\n" - "\nSpend the value of all (or specific) confirmed UTXOs in the wallet to one or more recipients.\n" + "\nSpend the value of all (or specific) confirmed UTXOs and unconfirmed change in the wallet to one or more recipients.\n" "Unconfirmed inbound UTXOs and locked UTXOs will not be spent. Sendall will respect the avoid_reuse wallet flag.\n" "If your wallet contains many small inputs, either because it received tiny payments or as a result of accumulating change, consider using `send_max` to exclude inputs that are worth less than the fees needed to spend them.\n", { @@ -1470,10 +1477,18 @@ RPCHelpMan sendall() } } + std::vector<COutPoint> outpoints_spent; + outpoints_spent.reserve(rawTx.vin.size()); + + for (const CTxIn& tx_in : rawTx.vin) { + outpoints_spent.push_back(tx_in.prevout); + } + // estimate final size of tx const TxSize tx_size{CalculateMaximumSignedTxSize(CTransaction(rawTx), pwallet.get())}; const CAmount fee_from_size{fee_rate.GetFee(tx_size.vsize)}; - const CAmount effective_value{total_input_value - fee_from_size}; + const std::optional<CAmount> total_bump_fees{pwallet->chain().calculateCombinedBumpFee(outpoints_spent, fee_rate)}; + CAmount effective_value = total_input_value - fee_from_size - total_bump_fees.value_or(0); if (fee_from_size > pwallet->m_default_max_tx_fee) { throw JSONRPCError(RPC_WALLET_ERROR, TransactionErrorString(TransactionError::MAX_FEE_EXCEEDED).original); @@ -1602,9 +1617,9 @@ RPCHelpMan walletprocesspsbt() if (sign) EnsureWalletIsUnlocked(*pwallet); - const TransactionError err{wallet.FillPSBT(psbtx, complete, nHashType, sign, bip32derivs, nullptr, finalize)}; - if (err != TransactionError::OK) { - throw JSONRPCTransactionError(err); + const auto err{wallet.FillPSBT(psbtx, complete, nHashType, sign, bip32derivs, nullptr, finalize)}; + if (err) { + throw JSONRPCPSBTError(*err); } UniValue result(UniValue::VOBJ); @@ -1736,9 +1751,9 @@ RPCHelpMan walletcreatefundedpsbt() // Fill transaction with out data but don't sign bool bip32derivs = request.params[4].isNull() ? true : request.params[4].get_bool(); bool complete = true; - const TransactionError err{wallet.FillPSBT(psbtx, complete, 1, /*sign=*/false, /*bip32derivs=*/bip32derivs)}; - if (err != TransactionError::OK) { - throw JSONRPCTransactionError(err); + const auto err{wallet.FillPSBT(psbtx, complete, 1, /*sign=*/false, /*bip32derivs=*/bip32derivs)}; + if (err) { + throw JSONRPCPSBTError(*err); } // Serialize the PSBT diff --git a/src/wallet/rpc/wallet.cpp b/src/wallet/rpc/wallet.cpp index 68331bf85f..8c218ad766 100644 --- a/src/wallet/rpc/wallet.cpp +++ b/src/wallet/rpc/wallet.cpp @@ -395,7 +395,7 @@ static RPCHelpMan createwallet() if (!request.params[4].isNull() && request.params[4].get_bool()) { flags |= WALLET_FLAG_AVOID_REUSE; } - if (self.Arg<bool>(5)) { + if (self.Arg<bool>("descriptors")) { #ifndef USE_SQLITE throw JSONRPCError(RPC_WALLET_ERROR, "Compiled without sqlite support (required for descriptor wallets)"); #endif @@ -489,7 +489,7 @@ static RPCHelpMan unloadwallet() // Release the "main" shared pointer and prevent further notifications. // Note that any attempt to load the same wallet would fail until the wallet // is destroyed (see CheckUniqueFileid). - std::optional<bool> load_on_start{self.MaybeArg<bool>(1)}; + std::optional<bool> load_on_start{self.MaybeArg<bool>("load_on_startup")}; if (!RemoveWallet(context, wallet, load_on_start, warnings)) { throw JSONRPCError(RPC_MISC_ERROR, "Requested wallet already unloaded"); } diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index b42275fe4b..c64aff5fe2 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -5,6 +5,7 @@ #include <hash.h> #include <key_io.h> #include <logging.h> +#include <node/types.h> #include <outputtype.h> #include <script/descriptor.h> #include <script/script.h> @@ -20,6 +21,9 @@ #include <optional> +using common::PSBTError; +using util::ToString; + namespace wallet { //! Value for the first BIP 32 hardened derivation. Can be used as a bit mask and as a value. See BIP 32 for more details. const uint32_t BIP32_HARDENED_KEY_LIMIT = 0x80000000; @@ -627,7 +631,7 @@ SigningResult LegacyScriptPubKeyMan::SignMessage(const std::string& message, con return SigningResult::SIGNING_FAILED; } -TransactionError LegacyScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbtx, const PrecomputedTransactionData& txdata, int sighash_type, bool sign, bool bip32derivs, int* n_signed, bool finalize) const +std::optional<PSBTError> LegacyScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbtx, const PrecomputedTransactionData& txdata, int sighash_type, bool sign, bool bip32derivs, int* n_signed, bool finalize) const { if (n_signed) { *n_signed = 0; @@ -642,13 +646,13 @@ TransactionError LegacyScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psb // Get the Sighash type if (sign && input.sighash_type != std::nullopt && *input.sighash_type != sighash_type) { - return TransactionError::SIGHASH_MISMATCH; + return PSBTError::SIGHASH_MISMATCH; } // Check non_witness_utxo has specified prevout if (input.non_witness_utxo) { if (txin.prevout.n >= input.non_witness_utxo->vout.size()) { - return TransactionError::MISSING_INPUTS; + return PSBTError::MISSING_INPUTS; } } else if (input.witness_utxo.IsNull()) { // There's no UTXO so we can just skip this now @@ -670,7 +674,7 @@ TransactionError LegacyScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psb UpdatePSBTOutput(HidingSigningProvider(this, true, !bip32derivs), psbtx, i); } - return TransactionError::OK; + return {}; } std::unique_ptr<CKeyMetadata> LegacyScriptPubKeyMan::GetMetadata(const CTxDestination& dest) const @@ -2485,7 +2489,7 @@ SigningResult DescriptorScriptPubKeyMan::SignMessage(const std::string& message, return SigningResult::OK; } -TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbtx, const PrecomputedTransactionData& txdata, int sighash_type, bool sign, bool bip32derivs, int* n_signed, bool finalize) const +std::optional<PSBTError> DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbtx, const PrecomputedTransactionData& txdata, int sighash_type, bool sign, bool bip32derivs, int* n_signed, bool finalize) const { if (n_signed) { *n_signed = 0; @@ -2500,7 +2504,7 @@ TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& // Get the Sighash type if (sign && input.sighash_type != std::nullopt && *input.sighash_type != sighash_type) { - return TransactionError::SIGHASH_MISMATCH; + return PSBTError::SIGHASH_MISMATCH; } // Get the scriptPubKey to know which SigningProvider to use @@ -2509,7 +2513,7 @@ TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& script = input.witness_utxo.scriptPubKey; } else if (input.non_witness_utxo) { if (txin.prevout.n >= input.non_witness_utxo->vout.size()) { - return TransactionError::MISSING_INPUTS; + return PSBTError::MISSING_INPUTS; } script = input.non_witness_utxo->vout[txin.prevout.n].scriptPubKey; } else { @@ -2580,7 +2584,7 @@ TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& UpdatePSBTOutput(HidingSigningProvider(keys.get(), /*hide_secret=*/true, /*hide_origin=*/!bip32derivs), psbtx, i); } - return TransactionError::OK; + return {}; } std::unique_ptr<CKeyMetadata> DescriptorScriptPubKeyMan::GetMetadata(const CTxDestination& dest) const diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 2c1ab8d44a..4d9f7bb1fa 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -6,13 +6,15 @@ #define BITCOIN_WALLET_SCRIPTPUBKEYMAN_H #include <addresstype.h> +#include <common/messages.h> +#include <common/signmessage.h> +#include <common/types.h> #include <logging.h> +#include <node/types.h> #include <psbt.h> #include <script/descriptor.h> #include <script/script.h> #include <script/signingprovider.h> -#include <util/error.h> -#include <util/message.h> #include <util/result.h> #include <util/time.h> #include <wallet/crypter.h> @@ -243,7 +245,7 @@ public: /** Sign a message with the given script */ virtual SigningResult SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const { return SigningResult::SIGNING_FAILED; }; /** Adds script and derivation path information to a PSBT, and optionally signs it. */ - virtual TransactionError FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = SIGHASH_DEFAULT, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const { return TransactionError::INVALID_PSBT; } + virtual std::optional<common::PSBTError> FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = SIGHASH_DEFAULT, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const { return common::PSBTError::UNSUPPORTED; } virtual uint256 GetID() const { return uint256(); } @@ -421,7 +423,7 @@ public: bool SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, bilingual_str>& input_errors) const override; SigningResult SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const override; - TransactionError FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = SIGHASH_DEFAULT, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const override; + std::optional<common::PSBTError> FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = SIGHASH_DEFAULT, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const override; uint256 GetID() const override; @@ -651,7 +653,7 @@ public: bool SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, bilingual_str>& input_errors) const override; SigningResult SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const override; - TransactionError FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = SIGHASH_DEFAULT, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const override; + std::optional<common::PSBTError> FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = SIGHASH_DEFAULT, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const override; uint256 GetID() const override; diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 9a7e166e68..4cbcfdb60f 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -4,10 +4,12 @@ #include <algorithm> #include <common/args.h> +#include <common/messages.h> #include <common/system.h> #include <consensus/amount.h> #include <consensus/validation.h> #include <interfaces/chain.h> +#include <node/types.h> #include <numeric> #include <policy/policy.h> #include <primitives/transaction.h> @@ -15,7 +17,6 @@ #include <script/signingprovider.h> #include <script/solver.h> #include <util/check.h> -#include <util/fees.h> #include <util/moneystr.h> #include <util/rbf.h> #include <util/trace.h> @@ -29,7 +30,10 @@ #include <cmath> +using common::StringForFeeReason; +using common::TransactionErrorString; using interfaces::FoundBlock; +using node::TransactionError; namespace wallet { static constexpr size_t OUTPUT_GROUP_MAX_ENTRIES{100}; @@ -132,7 +136,7 @@ static std::optional<int64_t> GetSignedTxinWeight(const CWallet* wallet, const C // txouts needs to be in the order of tx.vin TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, const std::vector<CTxOut>& txouts, const CCoinControl* coin_control) { - // nVersion + nLockTime + input count + output count + // version + nLockTime + input count + output count int64_t weight = (4 + 4 + GetSizeOfCompactSize(tx.vin.size()) + GetSizeOfCompactSize(tx.vout.size())) * WITNESS_SCALE_FACTOR; // Whether any input spends a witness program. Necessary to run before the next loop over the // inputs in order to accurately compute the compactSize length for the witness data per input. @@ -711,7 +715,7 @@ util::Result<SelectionResult> ChooseSelectionResult(interfaces::Chain& chain, co if (coin_selection_params.m_effective_feerate > CFeeRate{3 * coin_selection_params.m_long_term_feerate}) { // Minimize input set for feerates of at least 3×LTFRE (default: 30 ṩ/vB+) if (auto cg_result{CoinGrinder(groups.positive_group, nTargetValue, coin_selection_params.m_min_change_target, max_inputs_weight)}) { - cg_result->ComputeAndSetWaste(coin_selection_params.min_viable_change, coin_selection_params.m_cost_of_change, coin_selection_params.m_change_fee); + cg_result->RecalculateWaste(coin_selection_params.min_viable_change, coin_selection_params.m_cost_of_change, coin_selection_params.m_change_fee); results.push_back(*cg_result); } else { append_error(std::move(cg_result)); @@ -746,7 +750,7 @@ util::Result<SelectionResult> ChooseSelectionResult(interfaces::Chain& chain, co if (bump_fee_overestimate) { result.SetBumpFeeDiscount(bump_fee_overestimate); } - result.ComputeAndSetWaste(coin_selection_params.min_viable_change, coin_selection_params.m_cost_of_change, coin_selection_params.m_change_fee); + result.RecalculateWaste(coin_selection_params.min_viable_change, coin_selection_params.m_cost_of_change, coin_selection_params.m_change_fee); } // Choose the result with the least waste @@ -771,7 +775,7 @@ util::Result<SelectionResult> SelectCoins(const CWallet& wallet, CoinsResult& av if (selection_target <= 0) { SelectionResult result(nTargetValue, SelectionAlgorithm::MANUAL); result.AddInputs(pre_set_inputs.coins, coin_selection_params.m_subtract_fee_outputs); - result.ComputeAndSetWaste(coin_selection_params.min_viable_change, coin_selection_params.m_cost_of_change, coin_selection_params.m_change_fee); + result.RecalculateWaste(coin_selection_params.min_viable_change, coin_selection_params.m_cost_of_change, coin_selection_params.m_change_fee); return result; } @@ -792,7 +796,7 @@ util::Result<SelectionResult> SelectCoins(const CWallet& wallet, CoinsResult& av SelectionResult preselected(pre_set_inputs.total_amount, SelectionAlgorithm::MANUAL); preselected.AddInputs(pre_set_inputs.coins, coin_selection_params.m_subtract_fee_outputs); op_selection_result->Merge(preselected); - op_selection_result->ComputeAndSetWaste(coin_selection_params.min_viable_change, + op_selection_result->RecalculateWaste(coin_selection_params.min_viable_change, coin_selection_params.m_cost_of_change, coin_selection_params.m_change_fee); } @@ -985,7 +989,7 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( CMutableTransaction txNew; // The resulting transaction that we make if (coin_control.m_version) { - txNew.nVersion = coin_control.m_version.value(); + txNew.version = coin_control.m_version.value(); } CoinSelectionParams coin_selection_params{rng_fast}; // Parameters for coin selection, init with dummy @@ -1084,7 +1088,7 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( const auto change_spend_fee = coin_selection_params.m_discard_feerate.GetFee(coin_selection_params.change_spend_size); coin_selection_params.min_viable_change = std::max(change_spend_fee + 1, dust); - // Static vsize overhead + outputs vsize. 4 nVersion, 4 nLocktime, 1 input count, 1 witness overhead (dummy, flag, stack size) + // Static vsize overhead + outputs vsize. 4 version, 4 nLocktime, 1 input count, 1 witness overhead (dummy, flag, stack size) coin_selection_params.tx_noinputs_size = 10 + GetSizeOfCompactSize(vecSend.size()); // bytes for output count // vouts to the payees @@ -1394,7 +1398,7 @@ util::Result<CreatedTransactionResult> FundTransaction(CWallet& wallet, const CM coinControl.m_locktime = tx.nLockTime; // Set the user desired version - coinControl.m_version = tx.nVersion; + coinControl.m_version = tx.version; // Acquire the locks to prevent races to the new locked unspents between the // CreateTransaction call and LockCoin calls (when lockUnspents is true). diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 9a349f0992..7bd92b471c 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -874,29 +874,32 @@ BOOST_AUTO_TEST_CASE(SelectCoins_test) BOOST_AUTO_TEST_CASE(waste_test) { const CAmount fee{100}; + const CAmount min_viable_change{300}; const CAmount change_cost{125}; + const CAmount change_fee{30}; const CAmount fee_diff{40}; const CAmount in_amt{3 * COIN}; const CAmount target{2 * COIN}; - const CAmount excess{in_amt - fee * 2 - target}; + const CAmount excess{80}; + const CAmount exact_target{in_amt - fee * 2}; // Maximum spendable amount after fees: no change, no excess - // The following tests that the waste is calculated correctly in various scenarios. - // ComputeAndSetWaste will first determine the size of the change output. We don't really - // care about the change and just want to use the variant that always includes the change_cost, - // so min_viable_change and change_fee are set to 0 to ensure that. + // In the following, we test that the waste is calculated correctly in various scenarios. + // Usually, RecalculateWaste would compute change_fee and change_cost on basis of the + // change output type, current feerate, and discard_feerate, but we use fixed values + // across this test to make the test easier to understand. { // Waste with change is the change cost and difference between fee and long term fee SelectionResult selection1{target, SelectionAlgorithm::MANUAL}; - add_coin(1 * COIN, 1, selection1, fee, fee - fee_diff); + add_coin(1 * COIN, 1, selection1, /*fee=*/fee, /*long_term_fee=*/fee - fee_diff); add_coin(2 * COIN, 2, selection1, fee, fee - fee_diff); - selection1.ComputeAndSetWaste(/*min_viable_change=*/0, change_cost, /*change_fee=*/0); + selection1.RecalculateWaste(min_viable_change, change_cost, change_fee); BOOST_CHECK_EQUAL(fee_diff * 2 + change_cost, selection1.GetWaste()); // Waste will be greater when fee is greater, but long term fee is the same SelectionResult selection2{target, SelectionAlgorithm::MANUAL}; add_coin(1 * COIN, 1, selection2, fee * 2, fee - fee_diff); add_coin(2 * COIN, 2, selection2, fee * 2, fee - fee_diff); - selection2.ComputeAndSetWaste(/*min_viable_change=*/0, change_cost, /*change_fee=*/0); + selection2.RecalculateWaste(min_viable_change, change_cost, change_fee); BOOST_CHECK_GT(selection2.GetWaste(), selection1.GetWaste()); // Waste with change is the change cost and difference between fee and long term fee @@ -904,25 +907,25 @@ BOOST_AUTO_TEST_CASE(waste_test) SelectionResult selection3{target, SelectionAlgorithm::MANUAL}; add_coin(1 * COIN, 1, selection3, fee, fee + fee_diff); add_coin(2 * COIN, 2, selection3, fee, fee + fee_diff); - selection3.ComputeAndSetWaste(/*min_viable_change=*/0, change_cost, /*change_fee=*/0); + selection3.RecalculateWaste(min_viable_change, change_cost, change_fee); BOOST_CHECK_EQUAL(fee_diff * -2 + change_cost, selection3.GetWaste()); BOOST_CHECK_LT(selection3.GetWaste(), selection1.GetWaste()); } { // Waste without change is the excess and difference between fee and long term fee - SelectionResult selection_nochange1{target, SelectionAlgorithm::MANUAL}; + SelectionResult selection_nochange1{exact_target - excess, SelectionAlgorithm::MANUAL}; add_coin(1 * COIN, 1, selection_nochange1, fee, fee - fee_diff); add_coin(2 * COIN, 2, selection_nochange1, fee, fee - fee_diff); - selection_nochange1.ComputeAndSetWaste(/*min_viable_change=*/0, /*change_cost=*/0, /*change_fee=*/0); + selection_nochange1.RecalculateWaste(min_viable_change, change_cost, change_fee); BOOST_CHECK_EQUAL(fee_diff * 2 + excess, selection_nochange1.GetWaste()); // Waste without change is the excess and difference between fee and long term fee // With long term fee greater than fee, waste should be less than when long term fee is less than fee - SelectionResult selection_nochange2{target, SelectionAlgorithm::MANUAL}; + SelectionResult selection_nochange2{exact_target - excess, SelectionAlgorithm::MANUAL}; add_coin(1 * COIN, 1, selection_nochange2, fee, fee + fee_diff); add_coin(2 * COIN, 2, selection_nochange2, fee, fee + fee_diff); - selection_nochange2.ComputeAndSetWaste(/*min_viable_change=*/0, /*change_cost=*/0, /*change_fee=*/0); + selection_nochange2.RecalculateWaste(min_viable_change, change_cost, change_fee); BOOST_CHECK_EQUAL(fee_diff * -2 + excess, selection_nochange2.GetWaste()); BOOST_CHECK_LT(selection_nochange2.GetWaste(), selection_nochange1.GetWaste()); } @@ -932,57 +935,54 @@ BOOST_AUTO_TEST_CASE(waste_test) SelectionResult selection{target, SelectionAlgorithm::MANUAL}; add_coin(1 * COIN, 1, selection, fee, fee); add_coin(2 * COIN, 2, selection, fee, fee); - selection.ComputeAndSetWaste(/*min_viable_change=*/0, change_cost, /*change_fee=*/0); + selection.RecalculateWaste(min_viable_change, change_cost, change_fee); BOOST_CHECK_EQUAL(change_cost, selection.GetWaste()); } { // Waste without change and fee == long term fee is just the excess - SelectionResult selection{target, SelectionAlgorithm::MANUAL}; + SelectionResult selection{exact_target - excess, SelectionAlgorithm::MANUAL}; add_coin(1 * COIN, 1, selection, fee, fee); add_coin(2 * COIN, 2, selection, fee, fee); - selection.ComputeAndSetWaste(/*min_viable_change=*/0, /*change_cost=*/0, /*change_fee=*/0); + selection.RecalculateWaste(min_viable_change, change_cost, change_fee); BOOST_CHECK_EQUAL(excess, selection.GetWaste()); } { - // No Waste when fee == long_term_fee, no change, and no excess - const CAmount exact_target{in_amt - fee * 2}; + // Waste is 0 when fee == long_term_fee, no change, and no excess SelectionResult selection{exact_target, SelectionAlgorithm::MANUAL}; add_coin(1 * COIN, 1, selection, fee, fee); add_coin(2 * COIN, 2, selection, fee, fee); - selection.ComputeAndSetWaste(/*min_viable_change=*/0, /*change_cost=*/0, /*change_fee=*/0); + selection.RecalculateWaste(min_viable_change, change_cost , change_fee); BOOST_CHECK_EQUAL(0, selection.GetWaste()); } { - // No Waste when (fee - long_term_fee) == (-cost_of_change), and no excess + // Waste is 0 when (fee - long_term_fee) == (-cost_of_change), and no excess SelectionResult selection{target, SelectionAlgorithm::MANUAL}; - const CAmount new_change_cost{fee_diff * 2}; add_coin(1 * COIN, 1, selection, fee, fee + fee_diff); add_coin(2 * COIN, 2, selection, fee, fee + fee_diff); - selection.ComputeAndSetWaste(/*min_viable_change=*/0, new_change_cost, /*change_fee=*/0); + selection.RecalculateWaste(min_viable_change, /*change_cost=*/fee_diff * 2, change_fee); BOOST_CHECK_EQUAL(0, selection.GetWaste()); } { - // No Waste when (fee - long_term_fee) == (-excess), no change cost - const CAmount new_target{in_amt - fee * 2 - fee_diff * 2}; + // Waste is 0 when (fee - long_term_fee) == (-excess), no change cost + const CAmount new_target{exact_target - /*excess=*/fee_diff * 2}; SelectionResult selection{new_target, SelectionAlgorithm::MANUAL}; add_coin(1 * COIN, 1, selection, fee, fee + fee_diff); add_coin(2 * COIN, 2, selection, fee, fee + fee_diff); - selection.ComputeAndSetWaste(/*min_viable_change=*/0, /*change_cost=*/0, /*change_fee=*/0); + selection.RecalculateWaste(min_viable_change, change_cost, change_fee); BOOST_CHECK_EQUAL(0, selection.GetWaste()); } { // Negative waste when the long term fee is greater than the current fee and the selected value == target - const CAmount exact_target{3 * COIN - 2 * fee}; SelectionResult selection{exact_target, SelectionAlgorithm::MANUAL}; const CAmount target_waste1{-2 * fee_diff}; // = (2 * fee) - (2 * (fee + fee_diff)) add_coin(1 * COIN, 1, selection, fee, fee + fee_diff); add_coin(2 * COIN, 2, selection, fee, fee + fee_diff); - selection.ComputeAndSetWaste(/*min_viable_change=*/0, /*change_cost=*/0, /*change_fee=*/0); + selection.RecalculateWaste(min_viable_change, change_cost, change_fee); BOOST_CHECK_EQUAL(target_waste1, selection.GetWaste()); } @@ -990,10 +990,14 @@ BOOST_AUTO_TEST_CASE(waste_test) // Negative waste when the long term fee is greater than the current fee and change_cost < - (inputs * (fee - long_term_fee)) SelectionResult selection{target, SelectionAlgorithm::MANUAL}; const CAmount large_fee_diff{90}; - const CAmount target_waste2{-2 * large_fee_diff + change_cost}; // = (2 * fee) - (2 * (fee + large_fee_diff)) + change_cost + const CAmount target_waste2{-2 * large_fee_diff + change_cost}; + // = (2 * fee) - (2 * (fee + large_fee_diff)) + change_cost + // = (2 * 100) - (2 * (100 + 90)) + 125 + // = 200 - 380 + 125 = -55 + assert(target_waste2 == -55); add_coin(1 * COIN, 1, selection, fee, fee + large_fee_diff); add_coin(2 * COIN, 2, selection, fee, fee + large_fee_diff); - selection.ComputeAndSetWaste(/*min_viable_change=*/0, change_cost, /*change_fee=*/0); + selection.RecalculateWaste(min_viable_change, change_cost, change_fee); BOOST_CHECK_EQUAL(target_waste2, selection.GetWaste()); } } @@ -1018,12 +1022,12 @@ BOOST_AUTO_TEST_CASE(bump_fee_test) inputs[i]->ApplyBumpFee(20*(i+1)); } - selection.ComputeAndSetWaste(min_viable_change, change_cost, change_fee); + selection.RecalculateWaste(min_viable_change, change_cost, change_fee); CAmount expected_waste = fee_diff * -2 + change_cost + /*bump_fees=*/60; BOOST_CHECK_EQUAL(expected_waste, selection.GetWaste()); selection.SetBumpFeeDiscount(30); - selection.ComputeAndSetWaste(min_viable_change, change_cost, change_fee); + selection.RecalculateWaste(min_viable_change, change_cost, change_fee); expected_waste = fee_diff * -2 + change_cost + /*bump_fees=*/60 - /*group_discount=*/30; BOOST_CHECK_EQUAL(expected_waste, selection.GetWaste()); } @@ -1044,12 +1048,12 @@ BOOST_AUTO_TEST_CASE(bump_fee_test) inputs[i]->ApplyBumpFee(20*(i+1)); } - selection.ComputeAndSetWaste(min_viable_change, change_cost, change_fee); + selection.RecalculateWaste(min_viable_change, change_cost, change_fee); CAmount expected_waste = fee_diff * -2 + /*bump_fees=*/60 + /*excess = 100 - bump_fees*/40; BOOST_CHECK_EQUAL(expected_waste, selection.GetWaste()); selection.SetBumpFeeDiscount(30); - selection.ComputeAndSetWaste(min_viable_change, change_cost, change_fee); + selection.RecalculateWaste(min_viable_change, change_cost, change_fee); expected_waste = fee_diff * -2 + /*bump_fees=*/60 - /*group_discount=*/30 + /*excess = 100 - bump_fees + group_discount*/70; BOOST_CHECK_EQUAL(expected_waste, selection.GetWaste()); } @@ -1429,6 +1433,7 @@ BOOST_AUTO_TEST_CASE(check_max_weight) /*avoid_partial=*/false, }; + int max_weight = MAX_STANDARD_TX_WEIGHT - WITNESS_SCALE_FACTOR * (cs_params.tx_noinputs_size + cs_params.change_output_size); { // Scenario 1: // The actor starts with 1x 50.0 BTC and 1515x 0.033 BTC (~100.0 BTC total) unspent outputs @@ -1450,10 +1455,9 @@ BOOST_AUTO_TEST_CASE(check_max_weight) m_node); BOOST_CHECK(result); - // Verify that only the 50 BTC UTXO was selected - const auto& selection_res = result->GetInputSet(); - BOOST_CHECK(selection_res.size() == 1); - BOOST_CHECK((*selection_res.begin())->GetEffectiveValue() == 50 * COIN); + // Verify that the 50 BTC UTXO was selected, and result is below max_weight + BOOST_CHECK(has_coin(result->GetInputSet(), CAmount(50 * COIN))); + BOOST_CHECK_LE(result->GetWeight(), max_weight); } { @@ -1479,6 +1483,7 @@ BOOST_AUTO_TEST_CASE(check_max_weight) BOOST_CHECK(has_coin(result->GetInputSet(), CAmount(0.0625 * COIN))); BOOST_CHECK(has_coin(result->GetInputSet(), CAmount(0.025 * COIN))); + BOOST_CHECK_LE(result->GetWeight(), max_weight); } { diff --git a/src/wallet/test/fuzz/coinselection.cpp b/src/wallet/test/fuzz/coinselection.cpp index 331590df7f..644a8dd7ad 100644 --- a/src/wallet/test/fuzz/coinselection.cpp +++ b/src/wallet/test/fuzz/coinselection.cpp @@ -270,7 +270,7 @@ FUZZ_TARGET(coinselection) if (result_srd) { assert(result_srd->GetSelectedValue() >= target); assert(result_srd->GetChange(CHANGE_LOWER, coin_params.m_change_fee) > 0); // Demonstrate that SRD creates change of at least CHANGE_LOWER - result_srd->ComputeAndSetWaste(coin_params.min_viable_change, coin_params.m_cost_of_change, coin_params.m_change_fee); + result_srd->RecalculateWaste(coin_params.min_viable_change, coin_params.m_cost_of_change, coin_params.m_change_fee); (void)result_srd->GetShuffledInputVector(); (void)result_srd->GetInputSet(); } @@ -279,7 +279,7 @@ FUZZ_TARGET(coinselection) auto result_knapsack = KnapsackSolver(group_all, target, change_target, fast_random_context, MAX_STANDARD_TX_WEIGHT); if (result_knapsack) { assert(result_knapsack->GetSelectedValue() >= target); - result_knapsack->ComputeAndSetWaste(coin_params.min_viable_change, coin_params.m_cost_of_change, coin_params.m_change_fee); + result_knapsack->RecalculateWaste(coin_params.min_viable_change, coin_params.m_cost_of_change, coin_params.m_change_fee); (void)result_knapsack->GetShuffledInputVector(); (void)result_knapsack->GetInputSet(); } diff --git a/src/wallet/test/fuzz/crypter.cpp b/src/wallet/test/fuzz/crypter.cpp new file mode 100644 index 0000000000..62dd1bfde0 --- /dev/null +++ b/src/wallet/test/fuzz/crypter.cpp @@ -0,0 +1,92 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/setup_common.h> +#include <wallet/crypter.h> + +namespace wallet { +namespace { + +const TestingSetup* g_setup; +void initialize_crypter() +{ + static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); + g_setup = testing_setup.get(); +} + +FUZZ_TARGET(crypter, .init = initialize_crypter) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + bool good_data{true}; + + CCrypter crypt; + // These values are regularly updated within `CallOneOf` + std::vector<unsigned char> cipher_text_ed; + CKeyingMaterial plain_text_ed; + const std::vector<unsigned char> random_key = ConsumeRandomLengthByteVector(fuzzed_data_provider); + + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 10000) + { + CallOneOf( + fuzzed_data_provider, + [&] { + const std::string random_string = fuzzed_data_provider.ConsumeRandomLengthString(); + SecureString secure_string(random_string.begin(), random_string.end()); + + const unsigned int derivation_method = fuzzed_data_provider.ConsumeBool() ? 0 : fuzzed_data_provider.ConsumeIntegral<unsigned int>(); + + // Limiting the value of nRounds since it is otherwise uselessly expensive and causes a timeout when fuzzing. + crypt.SetKeyFromPassphrase(/*strKeyData=*/secure_string, + /*chSalt=*/ConsumeRandomLengthByteVector(fuzzed_data_provider), + /*nRounds=*/fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, 25000), + /*nDerivationMethod=*/derivation_method); + }, + [&] { + const std::vector<unsigned char> random_vector = ConsumeFixedLengthByteVector(fuzzed_data_provider, 32); + const CKeyingMaterial new_key(random_vector.begin(), random_vector.end()); + const std::vector<unsigned char>& new_IV = ConsumeFixedLengthByteVector(fuzzed_data_provider, 16); + crypt.SetKey(new_key, new_IV); + }, + [&] { + const std::vector<unsigned char> random_vector = ConsumeRandomLengthByteVector(fuzzed_data_provider); + plain_text_ed = CKeyingMaterial(random_vector.begin(), random_vector.end()); + }, + [&] { + cipher_text_ed = ConsumeRandomLengthByteVector(fuzzed_data_provider); + }, + [&] { + (void)crypt.Encrypt(plain_text_ed, cipher_text_ed); + }, + [&] { + (void)crypt.Decrypt(cipher_text_ed, plain_text_ed); + }, + [&] { + const CKeyingMaterial master_key(random_key.begin(), random_key.end()); + const uint256 iv = ConsumeUInt256(fuzzed_data_provider); + EncryptSecret(master_key, plain_text_ed, iv, cipher_text_ed); + }, + [&] { + const CKeyingMaterial master_key(random_key.begin(), random_key.end()); + const uint256 iv = ConsumeUInt256(fuzzed_data_provider); + DecryptSecret(master_key, cipher_text_ed, iv, plain_text_ed); + }, + [&] { + std::optional<CPubKey> random_pub_key = ConsumeDeserializable<CPubKey>(fuzzed_data_provider); + if (!random_pub_key) { + good_data = false; + return; + } + const CPubKey pub_key = *random_pub_key; + const CKeyingMaterial master_key(random_key.begin(), random_key.end()); + const std::vector<unsigned char> crypted_secret = ConsumeRandomLengthByteVector(fuzzed_data_provider); + CKey key; + DecryptKey(master_key, crypted_secret, pub_key, key); + }); + } +} +} // namespace +} // namespace wallet diff --git a/src/wallet/test/fuzz/scriptpubkeyman.cpp b/src/wallet/test/fuzz/scriptpubkeyman.cpp index 228e9629ed..835470aeae 100644 --- a/src/wallet/test/fuzz/scriptpubkeyman.cpp +++ b/src/wallet/test/fuzz/scriptpubkeyman.cpp @@ -137,6 +137,15 @@ FUZZ_TARGET(scriptpubkeyman, .init = initialize_spkm) PKHash{ConsumeUInt160(fuzzed_data_provider)}}; std::string str_sig; (void)spk_manager->SignMessage(msg, pk_hash, str_sig); + (void)spk_manager->GetMetadata(dest); + } + } + }, + [&] { + auto spks{spk_manager->GetScriptPubKeys()}; + for (const CScript& spk : spks) { + if (fuzzed_data_provider.ConsumeBool()) { + spk_manager->MarkUnusedAddresses(spk); } } }, @@ -148,6 +157,10 @@ FUZZ_TARGET(scriptpubkeyman, .init = initialize_spkm) } spk_manager->AddDescriptorKey(key, key.GetPubKey()); spk_manager->TopUp(); + LOCK(spk_manager->cs_desc_man); + auto particular_key{spk_manager->GetKey(key.GetPubKey().GetID())}; + assert(*particular_key == key); + assert(spk_manager->HasPrivKey(key.GetPubKey().GetID())); }, [&] { std::string descriptor; @@ -194,6 +207,9 @@ FUZZ_TARGET(scriptpubkeyman, .init = initialize_spkm) } ); } + + (void)spk_manager->GetEndRange(); + (void)spk_manager->GetKeyPoolSize(); } } // namespace diff --git a/src/wallet/test/fuzz/wallet_bdb_parser.cpp b/src/wallet/test/fuzz/wallet_bdb_parser.cpp index 5216e09769..6fbd695fc5 100644 --- a/src/wallet/test/fuzz/wallet_bdb_parser.cpp +++ b/src/wallet/test/fuzz/wallet_bdb_parser.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2023 The Bitcoin Core developers +// Copyright (c) 2023-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -63,6 +63,7 @@ FUZZ_TARGET(wallet_bdb_parser, .init = initialize_wallet_bdb_parser) #endif if (error.original.starts_with("AutoFile::ignore: end of file") || error.original.starts_with("AutoFile::read: end of file") || + error.original.starts_with("AutoFile::seek: ") || error.original == "Not a BDB file" || error.original == "Unexpected page type, should be 9 (BTree Metadata)" || error.original == "Unexpected database flags, should only be 0x20 (subdatabases)" || diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index 9f533bf6ed..b5a3b22c54 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <key_io.h> +#include <node/types.h> #include <util/bip32.h> #include <util/strencodings.h> #include <wallet/wallet.h> @@ -60,7 +61,7 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) // Fill transaction with our data bool complete = true; - BOOST_REQUIRE_EQUAL(TransactionError::OK, m_wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, false, true)); + BOOST_REQUIRE(!m_wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, false, true)); // Get the final tx DataStream ssTx{}; @@ -73,7 +74,7 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) // Try to sign the mutated input SignatureData sigdata; - BOOST_CHECK(m_wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, true, true) != TransactionError::OK); + BOOST_CHECK(m_wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, true, true)); } BOOST_AUTO_TEST_CASE(parse_hd_keypath) diff --git a/src/wallet/transaction.h b/src/wallet/transaction.h index 9c27574103..9079f6dd82 100644 --- a/src/wallet/transaction.h +++ b/src/wallet/transaction.h @@ -273,7 +273,7 @@ public: mapValueCopy["fromaccount"] = ""; if (nOrderPos != -1) { - mapValueCopy["n"] = ToString(nOrderPos); + mapValueCopy["n"] = util::ToString(nOrderPos); } if (nTimeSmart) { mapValueCopy["timesmart"] = strprintf("%u", nTimeSmart); diff --git a/src/wallet/types.h b/src/wallet/types.h index 6198f1ae33..7e3b2caeb1 100644 --- a/src/wallet/types.h +++ b/src/wallet/types.h @@ -3,12 +3,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -//! @file Public type definitions that are used inside and outside of the wallet -//! (e.g. by src/wallet and src/interfaces and src/qt code). +//! @file wallet/types.h is a home for public enum and struct type definitions +//! that are used by internally by wallet code, but also used externally by node +//! or GUI code. //! -//! File is home for simple enum and struct definitions that don't deserve -//! separate header files. More complicated wallet public types like -//! CCoinControl that are used externally can have separate headers. +//! This file is intended to define only simple types that do not have external +//! dependencies. More complicated public wallet types like CCoinControl should +//! be defined in dedicated header files. #ifndef BITCOIN_WALLET_TYPES_H #define BITCOIN_WALLET_TYPES_H diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 8a79cf730b..85cd67dab9 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -11,7 +11,9 @@ #include <chain.h> #include <coins.h> #include <common/args.h> +#include <common/messages.h> #include <common/settings.h> +#include <common/signmessage.h> #include <common/system.h> #include <consensus/amount.h> #include <consensus/consensus.h> @@ -25,6 +27,7 @@ #include <key.h> #include <key_io.h> #include <logging.h> +#include <node/types.h> #include <outputtype.h> #include <policy/feerate.h> #include <primitives/block.h> @@ -49,10 +52,8 @@ #include <uint256.h> #include <univalue.h> #include <util/check.h> -#include <util/error.h> #include <util/fs.h> #include <util/fs_helpers.h> -#include <util/message.h> #include <util/moneystr.h> #include <util/result.h> #include <util/string.h> @@ -81,7 +82,12 @@ struct KeyOriginInfo; +using common::AmountErrMsg; +using common::AmountHighWarn; +using common::PSBTError; using interfaces::FoundBlock; +using util::ReplaceAll; +using util::ToString; namespace wallet { @@ -2172,7 +2178,7 @@ bool CWallet::SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, return false; } -TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& complete, int sighash_type, bool sign, bool bip32derivs, size_t * n_signed, bool finalize) const +std::optional<PSBTError> CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& complete, int sighash_type, bool sign, bool bip32derivs, size_t * n_signed, bool finalize) const { if (n_signed) { *n_signed = 0; @@ -2205,9 +2211,9 @@ TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& comp // Fill in information from ScriptPubKeyMans for (ScriptPubKeyMan* spk_man : GetAllScriptPubKeyMans()) { int n_signed_this_spkm = 0; - TransactionError res = spk_man->FillPSBT(psbtx, txdata, sighash_type, sign, bip32derivs, &n_signed_this_spkm, finalize); - if (res != TransactionError::OK) { - return res; + const auto error{spk_man->FillPSBT(psbtx, txdata, sighash_type, sign, bip32derivs, &n_signed_this_spkm, finalize)}; + if (error) { + return error; } if (n_signed) { @@ -2223,7 +2229,7 @@ TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& comp complete &= PSBTInputSigned(input); } - return TransactionError::OK; + return {}; } SigningResult CWallet::SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 6a998fa398..5bc888462f 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -58,7 +58,9 @@ class Coin; class SigningProvider; enum class MemPoolRemovalReason; enum class SigningResult; -enum class TransactionError; +namespace common { +enum class PSBTError; +} // namespace common namespace interfaces { class Wallet; } @@ -659,7 +661,7 @@ public: * @param[in] finalize whether to create the final scriptSig or scriptWitness if possible * return error */ - TransactionError FillPSBT(PartiallySignedTransaction& psbtx, + std::optional<common::PSBTError> FillPSBT(PartiallySignedTransaction& psbtx, bool& complete, int sighash_type = SIGHASH_DEFAULT, bool sign = true, diff --git a/src/warnings.cpp b/src/warnings.cpp deleted file mode 100644 index 38c0554cf2..0000000000 --- a/src/warnings.cpp +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <config/bitcoin-config.h> // IWYU pragma: keep - -#include <warnings.h> - -#include <common/system.h> -#include <sync.h> -#include <util/translation.h> - -#include <optional> -#include <vector> - -static GlobalMutex g_warnings_mutex; -static bilingual_str g_misc_warnings GUARDED_BY(g_warnings_mutex); -static bool fLargeWorkInvalidChainFound GUARDED_BY(g_warnings_mutex) = false; -static std::optional<bilingual_str> g_timeoffset_warning GUARDED_BY(g_warnings_mutex){}; - -void SetMiscWarning(const bilingual_str& warning) -{ - LOCK(g_warnings_mutex); - g_misc_warnings = warning; -} - -void SetfLargeWorkInvalidChainFound(bool flag) -{ - LOCK(g_warnings_mutex); - fLargeWorkInvalidChainFound = flag; -} - -void SetMedianTimeOffsetWarning(std::optional<bilingual_str> warning) -{ - LOCK(g_warnings_mutex); - g_timeoffset_warning = warning; -} - -std::vector<bilingual_str> GetWarnings() -{ - std::vector<bilingual_str> warnings; - - LOCK(g_warnings_mutex); - - // Pre-release build warning - if (!CLIENT_VERSION_IS_RELEASE) { - warnings.emplace_back(_("This is a pre-release test build - use at your own risk - do not use for mining or merchant applications")); - } - - // Misc warnings like out of disk space and clock is wrong - if (!g_misc_warnings.empty()) { - warnings.emplace_back(g_misc_warnings); - } - - if (fLargeWorkInvalidChainFound) { - warnings.emplace_back(_("Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade.")); - } - - if (g_timeoffset_warning) { - warnings.emplace_back(g_timeoffset_warning.value()); - } - - return warnings; -} diff --git a/src/warnings.h b/src/warnings.h deleted file mode 100644 index 79dc2ffabf..0000000000 --- a/src/warnings.h +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_WARNINGS_H -#define BITCOIN_WARNINGS_H - -#include <optional> -#include <string> -#include <vector> - -struct bilingual_str; - -void SetMiscWarning(const bilingual_str& warning); -void SetfLargeWorkInvalidChainFound(bool flag); -/** Pass std::nullopt to disable the warning */ -void SetMedianTimeOffsetWarning(std::optional<bilingual_str> warning); -/** Return potential problems detected by the node. */ -std::vector<bilingual_str> GetWarnings(); - -#endif // BITCOIN_WARNINGS_H diff --git a/test/functional/feature_addrman.py b/test/functional/feature_addrman.py index 95d33d62ea..2efad70900 100755 --- a/test/functional/feature_addrman.py +++ b/test/functional/feature_addrman.py @@ -6,7 +6,6 @@ import os import re -import struct from test_framework.messages import ser_uint256, hash256, MAGIC_BYTES from test_framework.netutil import ADDRMAN_NEW_BUCKET_COUNT, ADDRMAN_TRIED_BUCKET_COUNT, ADDRMAN_BUCKET_SIZE @@ -28,15 +27,15 @@ def serialize_addrman( tried = [] INCOMPATIBILITY_BASE = 32 r = MAGIC_BYTES[net_magic] - r += struct.pack("B", format) - r += struct.pack("B", INCOMPATIBILITY_BASE + lowest_compatible) + r += format.to_bytes(1, "little") + r += (INCOMPATIBILITY_BASE + lowest_compatible).to_bytes(1, "little") r += ser_uint256(bucket_key) - r += struct.pack("<i", len_new or len(new)) - r += struct.pack("<i", len_tried or len(tried)) + r += (len_new or len(new)).to_bytes(4, "little", signed=True) + r += (len_tried or len(tried)).to_bytes(4, "little", signed=True) ADDRMAN_NEW_BUCKET_COUNT = 1 << 10 - r += struct.pack("<i", ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30)) + r += (ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30)).to_bytes(4, "little", signed=True) for _ in range(ADDRMAN_NEW_BUCKET_COUNT): - r += struct.pack("<i", 0) + r += (0).to_bytes(4, "little", signed=True) checksum = hash256(r) r += mock_checksum or checksum return r diff --git a/test/functional/feature_assumeutxo.py b/test/functional/feature_assumeutxo.py index 0b064aa32b..658eea0a0e 100755 --- a/test/functional/feature_assumeutxo.py +++ b/test/functional/feature_assumeutxo.py @@ -130,7 +130,8 @@ class AssumeutxoTest(BitcoinTestFramework): cases = [ # (content, offset, wrong_hash, custom_message) [b"\xff" * 32, 0, "7d52155c9a9fdc4525b637ef6170568e5dad6fabd0b1fdbb9432010b8453095b", None], # wrong outpoint hash - [(2).to_bytes(1, "little"), 32, None, "[snapshot] bad snapshot data after deserializing 1 coins"], # wrong outpoint hash + [(2).to_bytes(1, "little"), 32, None, "[snapshot] bad snapshot data after deserializing 1 coins"], # wrong txid coins count + [b"\xfd\xff\xff", 32, None, "[snapshot] mismatch in coins count in snapshot metadata and actual snapshot data"], # txid coins count exceeds coins left [b"\x01", 33, "9f4d897031ab8547665b4153317ae2fdbf0130c7840b66427ebc48b881cb80ad", None], # wrong outpoint index [b"\x81", 34, "3da966ba9826fb6d2604260e01607b55ba44e1a5de298606b08704bc62570ea8", None], # wrong coin code VARINT [b"\x80", 34, "091e893b3ccb4334378709578025356c8bcb0a623f37c7c4e493133c988648e5", None], # another wrong coin code @@ -195,6 +196,14 @@ class AssumeutxoTest(BitcoinTestFramework): path = node.datadir_path / node.chain / "invalid" / "path" assert_raises_rpc_error(-8, "Couldn't open file {} for reading.".format(path), node.loadtxoutset, path) + def test_snapshot_with_less_work(self, dump_output_path): + self.log.info("Test bitcoind should fail when snapshot has less accumulated work than this node.") + node = self.nodes[0] + assert_equal(node.getblockcount(), FINAL_HEIGHT) + with node.assert_debug_log(expected_msgs=["[snapshot] activation failed - work does not exceed active chainstate"]): + assert_raises_rpc_error(-32603, "Unable to load UTXO snapshot", node.loadtxoutset, dump_output_path) + self.restart_node(0, extra_args=self.extra_args[0]) + def run_test(self): """ Bring up two (disconnected) nodes, mine some new blocks on the first, @@ -276,6 +285,7 @@ class AssumeutxoTest(BitcoinTestFramework): assert_equal(n0.getblockchaininfo()["blocks"], FINAL_HEIGHT) + self.test_snapshot_with_less_work(dump_output['path']) self.test_invalid_mempool_state(dump_output['path']) self.test_invalid_snapshot_scenarios(dump_output['path']) self.test_invalid_chainstate_scenarios() diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index 8768d4040d..14b92d6733 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -78,8 +78,8 @@ class BIP68Test(BitcoinTestFramework): self.log.info("Activating BIP68 (and 112/113)") self.activateCSV() - self.log.info("Verifying nVersion=2 transactions are standard.") - self.log.info("Note that nVersion=2 transactions are always standard (independent of BIP68 activation status).") + self.log.info("Verifying version=2 transactions are standard.") + self.log.info("Note that version=2 transactions are always standard (independent of BIP68 activation status).") self.test_version2_relay() self.log.info("Passed") @@ -107,7 +107,7 @@ class BIP68Test(BitcoinTestFramework): # This transaction will enable sequence-locks, so this transaction should # fail tx2 = CTransaction() - tx2.nVersion = 2 + tx2.version = 2 sequence_value = sequence_value & 0x7fffffff tx2.vin = [CTxIn(COutPoint(tx1_id, 0), nSequence=sequence_value)] tx2.wit.vtxinwit = [CTxInWitness()] @@ -119,7 +119,7 @@ class BIP68Test(BitcoinTestFramework): # Setting the version back down to 1 should disable the sequence lock, # so this should be accepted. - tx2.nVersion = 1 + tx2.version = 1 self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=tx2.serialize().hex()) @@ -159,7 +159,7 @@ class BIP68Test(BitcoinTestFramework): using_sequence_locks = False tx = CTransaction() - tx.nVersion = 2 + tx.version = 2 value = 0 for j in range(num_inputs): sequence_value = 0xfffffffe # this disables sequence locks @@ -228,7 +228,7 @@ class BIP68Test(BitcoinTestFramework): # Anyone-can-spend mempool tx. # Sequence lock of 0 should pass. tx2 = CTransaction() - tx2.nVersion = 2 + tx2.version = 2 tx2.vin = [CTxIn(COutPoint(tx1.sha256, 0), nSequence=0)] tx2.vout = [CTxOut(int(tx1.vout[0].nValue - self.relayfee * COIN), SCRIPT_W0_SH_OP_TRUE)] self.wallet.sign_tx(tx=tx2) @@ -246,7 +246,7 @@ class BIP68Test(BitcoinTestFramework): sequence_value |= SEQUENCE_LOCKTIME_TYPE_FLAG tx = CTransaction() - tx.nVersion = 2 + tx.version = 2 tx.vin = [CTxIn(COutPoint(orig_tx.sha256, 0), nSequence=sequence_value)] tx.wit.vtxinwit = [CTxInWitness()] tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] @@ -360,7 +360,7 @@ class BIP68Test(BitcoinTestFramework): # Make an anyone-can-spend transaction tx2 = CTransaction() - tx2.nVersion = 1 + tx2.version = 1 tx2.vin = [CTxIn(COutPoint(tx1.sha256, 0), nSequence=0)] tx2.vout = [CTxOut(int(tx1.vout[0].nValue - self.relayfee * COIN), SCRIPT_W0_SH_OP_TRUE)] @@ -376,7 +376,7 @@ class BIP68Test(BitcoinTestFramework): sequence_value = 100 # 100 block relative locktime tx3 = CTransaction() - tx3.nVersion = 2 + tx3.version = 2 tx3.vin = [CTxIn(COutPoint(tx2.sha256, 0), nSequence=sequence_value)] tx3.wit.vtxinwit = [CTxInWitness()] tx3.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index 8a95975184..932f37a083 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test block processing.""" import copy -import struct import time from test_framework.blocktools import ( @@ -67,7 +66,7 @@ class CBrokenBlock(CBlock): def serialize(self, with_witness=False): r = b"" r += super(CBlock, self).serialize() - r += struct.pack("<BQ", 255, len(self.vtx)) + r += (255).to_bytes(1, "little") + len(self.vtx).to_bytes(8, "little") for tx in self.vtx: if with_witness: r += tx.serialize_with_witness() diff --git a/test/functional/feature_csv_activation.py b/test/functional/feature_csv_activation.py index bc1f9e8f2f..2db9682931 100755 --- a/test/functional/feature_csv_activation.py +++ b/test/functional/feature_csv_activation.py @@ -110,7 +110,7 @@ class BIP68_112_113Test(BitcoinTestFramework): def create_bip112special(self, input, txversion): tx = self.create_self_transfer_from_utxo(input) - tx.nVersion = txversion + tx.version = txversion self.miniwallet.sign_tx(tx) tx.vin[0].scriptSig = CScript([-1, OP_CHECKSEQUENCEVERIFY, OP_DROP] + list(CScript(tx.vin[0].scriptSig))) tx.rehash() @@ -118,7 +118,7 @@ class BIP68_112_113Test(BitcoinTestFramework): def create_bip112emptystack(self, input, txversion): tx = self.create_self_transfer_from_utxo(input) - tx.nVersion = txversion + tx.version = txversion self.miniwallet.sign_tx(tx) tx.vin[0].scriptSig = CScript([OP_CHECKSEQUENCEVERIFY] + list(CScript(tx.vin[0].scriptSig))) tx.rehash() @@ -136,7 +136,7 @@ class BIP68_112_113Test(BitcoinTestFramework): for i, (sdf, srhb, stf, srlb) in enumerate(product(*[[True, False]] * 4)): locktime = relative_locktime(sdf, srhb, stf, srlb) tx = self.create_self_transfer_from_utxo(bip68inputs[i]) - tx.nVersion = txversion + tx.version = txversion tx.vin[0].nSequence = locktime + locktime_delta self.miniwallet.sign_tx(tx) txs.append({'tx': tx, 'sdf': sdf, 'stf': stf}) @@ -154,7 +154,7 @@ class BIP68_112_113Test(BitcoinTestFramework): tx.vin[0].nSequence = BASE_RELATIVE_LOCKTIME + locktime_delta else: # vary nSequence instead, OP_CSV is fixed tx.vin[0].nSequence = locktime + locktime_delta - tx.nVersion = txversion + tx.version = txversion self.miniwallet.sign_tx(tx) if varyOP_CSV: tx.vin[0].scriptSig = CScript([locktime, OP_CHECKSEQUENCEVERIFY, OP_DROP] + list(CScript(tx.vin[0].scriptSig))) @@ -257,10 +257,10 @@ class BIP68_112_113Test(BitcoinTestFramework): # BIP113 test transaction will be modified before each use to put in appropriate block time bip113tx_v1 = self.create_self_transfer_from_utxo(bip113input) bip113tx_v1.vin[0].nSequence = 0xFFFFFFFE - bip113tx_v1.nVersion = 1 + bip113tx_v1.version = 1 bip113tx_v2 = self.create_self_transfer_from_utxo(bip113input) bip113tx_v2.vin[0].nSequence = 0xFFFFFFFE - bip113tx_v2.nVersion = 2 + bip113tx_v2.version = 2 # For BIP68 test all 16 relative sequence locktimes bip68txs_v1 = self.create_bip68txs(bip68inputs, 1) diff --git a/test/functional/feature_framework_miniwallet.py b/test/functional/feature_framework_miniwallet.py new file mode 100755 index 0000000000..f108289018 --- /dev/null +++ b/test/functional/feature_framework_miniwallet.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test MiniWallet.""" +from test_framework.blocktools import COINBASE_MATURITY +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_greater_than_or_equal, +) +from test_framework.wallet import ( + MiniWallet, + MiniWalletMode, +) + + +class FeatureFrameworkMiniWalletTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def test_tx_padding(self): + """Verify that MiniWallet's transaction padding (`target_weight` parameter) + works accurately enough (i.e. at most 3 WUs higher) with all modes.""" + for mode_name, wallet in self.wallets: + self.log.info(f"Test tx padding with MiniWallet mode {mode_name}...") + utxo = wallet.get_utxo(mark_as_spent=False) + for target_weight in [1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000, 4000000, + 989, 2001, 4337, 13371, 23219, 49153, 102035, 223419, 3999989]: + tx = wallet.create_self_transfer(utxo_to_spend=utxo, target_weight=target_weight)["tx"] + self.log.debug(f"-> target weight: {target_weight}, actual weight: {tx.get_weight()}") + assert_greater_than_or_equal(tx.get_weight(), target_weight) + assert_greater_than_or_equal(target_weight + 3, tx.get_weight()) + + def run_test(self): + node = self.nodes[0] + self.wallets = [ + ("ADDRESS_OP_TRUE", MiniWallet(node, mode=MiniWalletMode.ADDRESS_OP_TRUE)), + ("RAW_OP_TRUE", MiniWallet(node, mode=MiniWalletMode.RAW_OP_TRUE)), + ("RAW_P2PK", MiniWallet(node, mode=MiniWalletMode.RAW_P2PK)), + ] + for _, wallet in self.wallets: + self.generate(wallet, 10) + self.generate(wallet, COINBASE_MATURITY) + + self.test_tx_padding() + + +if __name__ == '__main__': + FeatureFrameworkMiniWalletTest().main() diff --git a/test/functional/feature_framework_unit_tests.py b/test/functional/feature_framework_unit_tests.py index f03f084bed..14d83f8a70 100755 --- a/test/functional/feature_framework_unit_tests.py +++ b/test/functional/feature_framework_unit_tests.py @@ -27,6 +27,7 @@ TEST_FRAMEWORK_MODULES = [ "crypto.ripemd160", "crypto.secp256k1", "script", + "script_util", "segwit_addr", "wallet_util", ] diff --git a/test/functional/feature_reindex.py b/test/functional/feature_reindex.py index f0f32a61ab..835cd0c5cf 100755 --- a/test/functional/feature_reindex.py +++ b/test/functional/feature_reindex.py @@ -73,6 +73,25 @@ class ReindexTest(BitcoinTestFramework): # All blocks should be accepted and processed. assert_equal(self.nodes[0].getblockcount(), 12) + def continue_reindex_after_shutdown(self): + node = self.nodes[0] + self.generate(node, 1500) + + # Restart node with reindex and stop reindex as soon as it starts reindexing + self.log.info("Restarting node while reindexing..") + node.stop_node() + with node.busy_wait_for_debug_log([b'initload thread start']): + node.start(['-blockfilterindex', '-reindex']) + node.wait_for_rpc_connection(wait_for_import=False) + node.stop_node() + + # Start node without the reindex flag and verify it does not wipe the indexes data again + db_path = node.chain_path / 'indexes' / 'blockfilter' / 'basic' / 'db' + with node.assert_debug_log(expected_msgs=[f'Opening LevelDB in {db_path}'], unexpected_msgs=[f'Wiping LevelDB in {db_path}']): + node.start(['-blockfilterindex']) + node.wait_for_rpc_connection(wait_for_import=False) + node.stop_node() + def run_test(self): self.reindex(False) self.reindex(True) @@ -80,6 +99,7 @@ class ReindexTest(BitcoinTestFramework): self.reindex(True) self.out_of_order() + self.continue_reindex_after_shutdown() if __name__ == '__main__': diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index e7d65b4539..1a0844d240 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -1408,10 +1408,10 @@ class TaprootTest(BitcoinTestFramework): left = done while left: - # Construct CTransaction with random nVersion, nLocktime + # Construct CTransaction with random version, nLocktime tx = CTransaction() - tx.nVersion = random.choice([1, 2, random.randint(-0x80000000, 0x7fffffff)]) - min_sequence = (tx.nVersion != 1 and tx.nVersion != 0) * 0x80000000 # The minimum sequence number to disable relative locktime + tx.version = random.choice([1, 2, random.getrandbits(32)]) + min_sequence = (tx.version != 1 and tx.version != 0) * 0x80000000 # The minimum sequence number to disable relative locktime if random.choice([True, False]): tx.nLockTime = random.randrange(LOCKTIME_THRESHOLD, self.lastblocktime - 7200) # all absolute locktimes in the past else: @@ -1502,8 +1502,8 @@ class TaprootTest(BitcoinTestFramework): is_standard_tx = ( fail_input is None # Must be valid to be standard and (all(utxo.spender.is_standard for utxo in input_utxos)) # All inputs must be standard - and tx.nVersion >= 1 # The tx version must be standard - and tx.nVersion <= 2) + and tx.version >= 1 # The tx version must be standard + and tx.version <= 2) tx.rehash() msg = ','.join(utxo.spender.comment + ("*" if n == fail_input else "") for n, utxo in enumerate(input_utxos)) if is_standard_tx: @@ -1530,7 +1530,7 @@ class TaprootTest(BitcoinTestFramework): # Deterministically mine coins to OP_TRUE in block 1 assert_equal(self.nodes[0].getblockcount(), 0) coinbase = CTransaction() - coinbase.nVersion = 1 + coinbase.version = 1 coinbase.vin = [CTxIn(COutPoint(0, 0xffffffff), CScript([OP_1, OP_1]), SEQUENCE_FINAL)] coinbase.vout = [CTxOut(5000000000, CScript([OP_1]))] coinbase.nLockTime = 0 @@ -1622,7 +1622,7 @@ class TaprootTest(BitcoinTestFramework): for i, spk in enumerate(old_spks + tap_spks): val = 42000000 * (i + 7) tx = CTransaction() - tx.nVersion = 1 + tx.version = 1 tx.vin = [CTxIn(COutPoint(lasttxid, i & 1), CScript([]), SEQUENCE_FINAL)] tx.vout = [CTxOut(val, spk), CTxOut(amount - val, CScript([OP_1]))] if i & 1: @@ -1679,7 +1679,7 @@ class TaprootTest(BitcoinTestFramework): # Construct a deterministic transaction spending all outputs created above. tx = CTransaction() - tx.nVersion = 2 + tx.version = 2 tx.vin = [] inputs = [] input_spks = [tap_spks[0], tap_spks[1], old_spks[0], tap_spks[2], tap_spks[5], old_spks[2], tap_spks[6], tap_spks[3], tap_spks[4]] diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index 83bb5121e5..a6628dcbf3 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -8,6 +8,7 @@ from decimal import Decimal import re from test_framework.blocktools import COINBASE_MATURITY +from test_framework.netutil import test_ipv6_local from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -15,6 +16,7 @@ from test_framework.util import ( assert_raises_process_error, assert_raises_rpc_error, get_auth_cookie, + rpc_port, ) import time @@ -107,6 +109,53 @@ class TestBitcoinCli(BitcoinTestFramework): self.log.info("Test connecting to a non-existing server") assert_raises_process_error(1, "Could not connect to the server", self.nodes[0].cli('-rpcport=1').echo) + self.log.info("Test handling of invalid ports in rpcconnect") + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: 127.0.0.1:notaport", self.nodes[0].cli("-rpcconnect=127.0.0.1:notaport").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: 127.0.0.1:-1", self.nodes[0].cli("-rpcconnect=127.0.0.1:-1").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: 127.0.0.1:0", self.nodes[0].cli("-rpcconnect=127.0.0.1:0").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: 127.0.0.1:65536", self.nodes[0].cli("-rpcconnect=127.0.0.1:65536").echo) + + self.log.info("Checking for IPv6") + have_ipv6 = test_ipv6_local() + if not have_ipv6: + self.log.info("Skipping IPv6 tests") + + if have_ipv6: + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: [::1]:notaport", self.nodes[0].cli("-rpcconnect=[::1]:notaport").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: [::1]:-1", self.nodes[0].cli("-rpcconnect=[::1]:-1").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: [::1]:0", self.nodes[0].cli("-rpcconnect=[::1]:0").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: [::1]:65536", self.nodes[0].cli("-rpcconnect=[::1]:65536").echo) + + self.log.info("Test handling of invalid ports in rpcport") + assert_raises_process_error(1, "Invalid port provided in -rpcport: notaport", self.nodes[0].cli("-rpcport=notaport").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcport: -1", self.nodes[0].cli("-rpcport=-1").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcport: 0", self.nodes[0].cli("-rpcport=0").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcport: 65536", self.nodes[0].cli("-rpcport=65536").echo) + + self.log.info("Test port usage preferences") + node_rpc_port = rpc_port(self.nodes[0].index) + # Prevent bitcoin-cli from using existing rpcport in conf + conf_rpcport = "rpcport=" + str(node_rpc_port) + self.nodes[0].replace_in_config([(conf_rpcport, "#" + conf_rpcport)]) + # prefer rpcport over rpcconnect + assert_raises_process_error(1, "Could not connect to the server 127.0.0.1:1", self.nodes[0].cli(f"-rpcconnect=127.0.0.1:{node_rpc_port}", "-rpcport=1").echo) + if have_ipv6: + assert_raises_process_error(1, "Could not connect to the server ::1:1", self.nodes[0].cli(f"-rpcconnect=[::1]:{node_rpc_port}", "-rpcport=1").echo) + + assert_equal(BLOCKS, self.nodes[0].cli("-rpcconnect=127.0.0.1:18999", f'-rpcport={node_rpc_port}').getblockcount()) + if have_ipv6: + assert_equal(BLOCKS, self.nodes[0].cli("-rpcconnect=[::1]:18999", f'-rpcport={node_rpc_port}').getblockcount()) + + # prefer rpcconnect port over default + assert_equal(BLOCKS, self.nodes[0].cli(f"-rpcconnect=127.0.0.1:{node_rpc_port}").getblockcount()) + if have_ipv6: + assert_equal(BLOCKS, self.nodes[0].cli(f"-rpcconnect=[::1]:{node_rpc_port}").getblockcount()) + + # prefer rpcport over default + assert_equal(BLOCKS, self.nodes[0].cli(f'-rpcport={node_rpc_port}').getblockcount()) + # Re-enable rpcport in conf if present + self.nodes[0].replace_in_config([("#" + conf_rpcport, conf_rpcport)]) + self.log.info("Test connecting with non-existing RPC cookie file") assert_raises_process_error(1, "Could not locate RPC credentials", self.nodes[0].cli('-rpccookiefile=does-not-exist', '-rpcpassword=').echo) diff --git a/test/functional/interface_rpc.py b/test/functional/interface_rpc.py index b08ca42796..6c1855c400 100755 --- a/test/functional/interface_rpc.py +++ b/test/functional/interface_rpc.py @@ -14,7 +14,6 @@ from typing import Optional import subprocess -RPC_INVALID_ADDRESS_OR_KEY = -5 RPC_INVALID_PARAMETER = -8 RPC_METHOD_NOT_FOUND = -32601 RPC_INVALID_REQUEST = -32600 diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index b00be5f4f0..3d205ffa62 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -287,7 +287,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.log.info('Some nonstandard transactions') tx = tx_from_hex(raw_tx_reference) - tx.nVersion = 3 # A version currently non-standard + tx.version = 4 # A version currently non-standard self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'version'}], rawtxs=[tx.serialize().hex()], diff --git a/test/functional/mempool_accept_v3.py b/test/functional/mempool_accept_v3.py index 6b0ba0812b..d4a33c232e 100755 --- a/test/functional/mempool_accept_v3.py +++ b/test/functional/mempool_accept_v3.py @@ -42,7 +42,7 @@ def cleanup(extra_args=None): class MempoolAcceptV3(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.extra_args = [["-acceptnonstdtxn=1"]] + self.extra_args = [[]] self.setup_clean_chain = True def check_mempool(self, txids): @@ -51,7 +51,7 @@ class MempoolAcceptV3(BitcoinTestFramework): assert_equal(len(txids), len(mempool_contents)) assert all([txid in txids for txid in mempool_contents]) - @cleanup(extra_args=["-datacarriersize=20000", "-acceptnonstdtxn=1"]) + @cleanup(extra_args=["-datacarriersize=20000"]) def test_v3_max_vsize(self): node = self.nodes[0] self.log.info("Test v3-specific maximum transaction vsize") @@ -65,7 +65,7 @@ class MempoolAcceptV3(BitcoinTestFramework): tx_v2_heavy = self.wallet.send_self_transfer(from_node=node, target_weight=(V3_MAX_VSIZE + 1) * WITNESS_SCALE_FACTOR, version=2) self.check_mempool([tx_v2_heavy["txid"]]) - @cleanup(extra_args=["-datacarriersize=1000", "-acceptnonstdtxn=1"]) + @cleanup(extra_args=["-datacarriersize=1000"]) def test_v3_acceptance(self): node = self.nodes[0] self.log.info("Test a child of a v3 transaction cannot be more than 1000vB") @@ -105,7 +105,7 @@ class MempoolAcceptV3(BitcoinTestFramework): self.check_mempool([tx_v3_parent_normal["txid"], tx_v3_child_almost_heavy_rbf["txid"]]) assert_equal(node.getmempoolentry(tx_v3_parent_normal["txid"])["descendantcount"], 2) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) + @cleanup(extra_args=None) def test_v3_replacement(self): node = self.nodes[0] self.log.info("Test v3 transactions may be replaced by v3 transactions") @@ -162,7 +162,7 @@ class MempoolAcceptV3(BitcoinTestFramework): self.check_mempool([tx_v3_bip125_rbf_v2["txid"], tx_v3_parent["txid"], tx_v3_child["txid"]]) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) + @cleanup(extra_args=None) def test_v3_bip125(self): node = self.nodes[0] self.log.info("Test v3 transactions that don't signal BIP125 are replaceable") @@ -186,7 +186,7 @@ class MempoolAcceptV3(BitcoinTestFramework): ) self.check_mempool([tx_v3_no_bip125_rbf["txid"]]) - @cleanup(extra_args=["-datacarriersize=40000", "-acceptnonstdtxn=1"]) + @cleanup(extra_args=["-datacarriersize=40000"]) def test_v3_reorg(self): node = self.nodes[0] self.log.info("Test that, during a reorg, v3 rules are not enforced") @@ -208,7 +208,7 @@ class MempoolAcceptV3(BitcoinTestFramework): node.reconsiderblock(block[0]) - @cleanup(extra_args=["-limitdescendantsize=10", "-datacarriersize=40000", "-acceptnonstdtxn=1"]) + @cleanup(extra_args=["-limitdescendantsize=10", "-datacarriersize=40000"]) def test_nondefault_package_limits(self): """ Max standard tx size + v3 rules imply the ancestor/descendant rules (at their default @@ -241,7 +241,7 @@ class MempoolAcceptV3(BitcoinTestFramework): self.generate(node, 1) self.log.info("Test that a decreased limitancestorsize also applies to v3 parent") - self.restart_node(0, extra_args=["-limitancestorsize=10", "-datacarriersize=40000", "-acceptnonstdtxn=1"]) + self.restart_node(0, extra_args=["-limitancestorsize=10", "-datacarriersize=40000"]) tx_v3_parent_large2 = self.wallet.send_self_transfer( from_node=node, target_weight=parent_target_weight, @@ -261,7 +261,7 @@ class MempoolAcceptV3(BitcoinTestFramework): assert_raises_rpc_error(-26, f"too-long-mempool-chain, exceeds ancestor size limit", node.sendrawtransaction, tx_v3_child_large2["hex"]) self.check_mempool([tx_v3_parent_large2["txid"]]) - @cleanup(extra_args=["-datacarriersize=1000", "-acceptnonstdtxn=1"]) + @cleanup(extra_args=["-datacarriersize=1000"]) def test_v3_ancestors_package(self): self.log.info("Test that v3 ancestor limits are checked within the package") node = self.nodes[0] @@ -304,7 +304,7 @@ class MempoolAcceptV3(BitcoinTestFramework): result = node.testmempoolaccept([tx_v3_parent["hex"], tx_v3_child["hex"], tx_v3_grandchild["hex"]]) assert all([txresult["package-error"] == f"v3-violation, tx {tx_v3_grandchild['txid']} (wtxid={tx_v3_grandchild['wtxid']}) would have too many ancestors" for txresult in result]) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) + @cleanup(extra_args=None) def test_v3_ancestors_package_and_mempool(self): """ A v3 transaction in a package cannot have 2 v3 parents. @@ -334,7 +334,7 @@ class MempoolAcceptV3(BitcoinTestFramework): assert_equal(result['package_msg'], f"v3-violation, tx {tx_child_violator['txid']} (wtxid={tx_child_violator['wtxid']}) would have too many ancestors") self.check_mempool([tx_in_mempool["txid"]]) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) + @cleanup(extra_args=None) def test_sibling_eviction_package(self): """ When a transaction has a mempool sibling, it may be eligible for sibling eviction. @@ -410,7 +410,7 @@ class MempoolAcceptV3(BitcoinTestFramework): assert_equal(result_package_cpfp["tx-results"][tx_sibling_3['wtxid']]['error'], expected_error_cpfp) - @cleanup(extra_args=["-datacarriersize=1000", "-acceptnonstdtxn=1"]) + @cleanup(extra_args=["-datacarriersize=1000"]) def test_v3_package_inheritance(self): self.log.info("Test that v3 inheritance is checked within package") node = self.nodes[0] @@ -429,7 +429,7 @@ class MempoolAcceptV3(BitcoinTestFramework): assert_equal(result['package_msg'], f"v3-violation, non-v3 tx {tx_v2_child['txid']} (wtxid={tx_v2_child['wtxid']}) cannot spend from v3 tx {tx_v3_parent['txid']} (wtxid={tx_v3_parent['wtxid']})") self.check_mempool([]) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) + @cleanup(extra_args=None) def test_v3_in_testmempoolaccept(self): node = self.nodes[0] @@ -479,7 +479,7 @@ class MempoolAcceptV3(BitcoinTestFramework): test_accept_2children_with_in_mempool_parent = node.testmempoolaccept([tx_v3_child_1["hex"], tx_v3_child_2["hex"]]) assert all([result["package-error"] == expected_error_extra for result in test_accept_2children_with_in_mempool_parent]) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) + @cleanup(extra_args=None) def test_reorg_2child_rbf(self): node = self.nodes[0] self.log.info("Test that children of a v3 transaction can be replaced individually, even if there are multiple due to reorg") @@ -510,7 +510,7 @@ class MempoolAcceptV3(BitcoinTestFramework): self.check_mempool([ancestor_tx["txid"], child_1_conflict["txid"], child_2["txid"]]) assert_equal(node.getmempoolentry(ancestor_tx["txid"])["descendantcount"], 3) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) + @cleanup(extra_args=None) def test_v3_sibling_eviction(self): self.log.info("Test sibling eviction for v3") node = self.nodes[0] @@ -583,7 +583,7 @@ class MempoolAcceptV3(BitcoinTestFramework): node.sendrawtransaction(tx_v3_child_3["hex"]) self.check_mempool(txids_v2_100 + [tx_v3_parent["txid"], tx_v3_child_3["txid"]]) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) + @cleanup(extra_args=None) def test_reorg_sibling_eviction_1p2c(self): node = self.nodes[0] self.log.info("Test that sibling eviction is not allowed when multiple siblings exist") diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index d46924f4ce..49a0a32c45 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -59,7 +59,7 @@ class MempoolLimitTest(BitcoinTestFramework): mempoolmin_feerate = node.getmempoolinfo()["mempoolminfee"] tx_A = self.wallet.send_self_transfer( from_node=node, - fee=(mempoolmin_feerate / 1000) * (A_weight // 4) + Decimal('0.000001'), + fee_rate=mempoolmin_feerate, target_weight=A_weight, utxo_to_spend=rbf_utxo, confirmed_only=True @@ -77,7 +77,7 @@ class MempoolLimitTest(BitcoinTestFramework): non_cpfp_carveout_weight = 40001 # EXTRA_DESCENDANT_TX_SIZE_LIMIT + 1 tx_C = self.wallet.create_self_transfer( target_weight=non_cpfp_carveout_weight, - fee = (mempoolmin_feerate / 1000) * (non_cpfp_carveout_weight // 4) + Decimal('0.000001'), + fee_rate=mempoolmin_feerate, utxo_to_spend=tx_B["new_utxo"], confirmed_only=True ) @@ -109,7 +109,7 @@ class MempoolLimitTest(BitcoinTestFramework): # happen in the middle of package evaluation, as it can invalidate the coins cache. mempool_evicted_tx = self.wallet.send_self_transfer( from_node=node, - fee=(mempoolmin_feerate / 1000) * (evicted_weight // 4) + Decimal('0.000001'), + fee_rate=mempoolmin_feerate, target_weight=evicted_weight, confirmed_only=True ) @@ -135,11 +135,11 @@ class MempoolLimitTest(BitcoinTestFramework): parent_weight = 100000 num_big_parents = 3 assert_greater_than(parent_weight * num_big_parents, current_info["maxmempool"] - current_info["bytes"]) - parent_fee = (100 * mempoolmin_feerate / 1000) * (parent_weight // 4) + parent_feerate = 100 * mempoolmin_feerate big_parent_txids = [] for i in range(num_big_parents): - parent = self.wallet.create_self_transfer(fee=parent_fee, target_weight=parent_weight, confirmed_only=True) + parent = self.wallet.create_self_transfer(fee_rate=parent_feerate, target_weight=parent_weight, confirmed_only=True) parent_utxos.append(parent["new_utxo"]) package_hex.append(parent["hex"]) big_parent_txids.append(parent["txid"]) @@ -314,18 +314,20 @@ class MempoolLimitTest(BitcoinTestFramework): target_weight_each = 200000 assert_greater_than(target_weight_each * 2, node.getmempoolinfo()["maxmempool"] - node.getmempoolinfo()["bytes"]) # Should be a true CPFP: parent's feerate is just below mempool min feerate - parent_fee = (mempoolmin_feerate / 1000) * (target_weight_each // 4) - Decimal("0.00001") + parent_feerate = mempoolmin_feerate - Decimal("0.000001") # 0.1 sats/vbyte below min feerate # Parent + child is above mempool minimum feerate - child_fee = (worst_feerate_btcvb) * (target_weight_each // 4) - Decimal("0.00001") + child_feerate = (worst_feerate_btcvb * 1000) - Decimal("0.000001") # 0.1 sats/vbyte below worst feerate # However, when eviction is triggered, these transactions should be at the bottom. # This assertion assumes parent and child are the same size. miniwallet.rescan_utxos() - tx_parent_just_below = miniwallet.create_self_transfer(fee=parent_fee, target_weight=target_weight_each) - tx_child_just_above = miniwallet.create_self_transfer(utxo_to_spend=tx_parent_just_below["new_utxo"], fee=child_fee, target_weight=target_weight_each) + tx_parent_just_below = miniwallet.create_self_transfer(fee_rate=parent_feerate, target_weight=target_weight_each) + tx_child_just_above = miniwallet.create_self_transfer(utxo_to_spend=tx_parent_just_below["new_utxo"], fee_rate=child_feerate, target_weight=target_weight_each) # This package ranks below the lowest descendant package in the mempool - assert_greater_than(worst_feerate_btcvb, (parent_fee + child_fee) / (tx_parent_just_below["tx"].get_vsize() + tx_child_just_above["tx"].get_vsize())) - assert_greater_than(mempoolmin_feerate, (parent_fee) / (tx_parent_just_below["tx"].get_vsize())) - assert_greater_than((parent_fee + child_fee) / (tx_parent_just_below["tx"].get_vsize() + tx_child_just_above["tx"].get_vsize()), mempoolmin_feerate / 1000) + package_fee = tx_parent_just_below["fee"] + tx_child_just_above["fee"] + package_vsize = tx_parent_just_below["tx"].get_vsize() + tx_child_just_above["tx"].get_vsize() + assert_greater_than(worst_feerate_btcvb, package_fee / package_vsize) + assert_greater_than(mempoolmin_feerate, tx_parent_just_below["fee"] / (tx_parent_just_below["tx"].get_vsize())) + assert_greater_than(package_fee / package_vsize, mempoolmin_feerate / 1000) res = node.submitpackage([tx_parent_just_below["hex"], tx_child_just_above["hex"]]) for wtxid in [tx_parent_just_below["wtxid"], tx_child_just_above["wtxid"]]: assert_equal(res["tx-results"][wtxid]["error"], "mempool full") diff --git a/test/functional/mempool_package_onemore.py b/test/functional/mempool_package_onemore.py index 8f58b67110..632425814a 100755 --- a/test/functional/mempool_package_onemore.py +++ b/test/functional/mempool_package_onemore.py @@ -40,7 +40,7 @@ class MempoolPackagesTest(BitcoinTestFramework): for _ in range(DEFAULT_ANCESTOR_LIMIT - 4): utxo, = self.chain_tx([utxo]) chain.append(utxo) - second_chain, = self.chain_tx([self.wallet.get_utxo()]) + second_chain, = self.chain_tx([self.wallet.get_utxo(confirmed_only=True)]) # Check mempool has DEFAULT_ANCESTOR_LIMIT + 1 transactions in it assert_equal(len(self.nodes[0].getrawmempool()), DEFAULT_ANCESTOR_LIMIT + 1) diff --git a/test/functional/mempool_package_rbf.py b/test/functional/mempool_package_rbf.py new file mode 100755 index 0000000000..ceb9530394 --- /dev/null +++ b/test/functional/mempool_package_rbf.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from decimal import Decimal + +from test_framework.messages import ( + COIN, + MAX_BIP125_RBF_SEQUENCE, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.mempool_util import fill_mempool +from test_framework.util import ( + assert_greater_than_or_equal, + assert_equal, +) +from test_framework.wallet import ( + DEFAULT_FEE, + MiniWallet, +) + +MAX_REPLACEMENT_CANDIDATES = 100 + +# Value high enough to cause evictions in each subtest +# for typical cases +DEFAULT_CHILD_FEE = DEFAULT_FEE * 4 + +class PackageRBFTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.setup_clean_chain = True + # Required for fill_mempool() + self.extra_args = [[ + "-datacarriersize=100000", + "-maxmempool=5", + ]] * self.num_nodes + + def assert_mempool_contents(self, expected=None): + """Assert that all transactions in expected are in the mempool, + and no additional ones exist. + """ + if not expected: + expected = [] + mempool = self.nodes[0].getrawmempool(verbose=False) + assert_equal(len(mempool), len(expected)) + for tx in expected: + assert tx.rehash() in mempool + + def create_simple_package(self, parent_coin, parent_fee=DEFAULT_FEE, child_fee=DEFAULT_CHILD_FEE, heavy_child=False): + """Create a 1 parent 1 child package using the coin passed in as the parent's input. The + parent has 1 output, used to fund 1 child transaction. + All transactions signal BIP125 replaceability, but nSequence changes based on self.ctr. This + prevents identical txids between packages when the parents spend the same coin and have the + same fee (i.e. 0sat). + + returns tuple (hex serialized txns, CTransaction objects) + """ + self.ctr += 1 + # Use fee_rate=0 because create_self_transfer will use the default fee_rate value otherwise. + # Passing in fee>0 overrides fee_rate, so this still works for non-zero parent_fee. + parent_result = self.wallet.create_self_transfer( + fee=parent_fee, + utxo_to_spend=parent_coin, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + num_child_outputs = 10 if heavy_child else 1 + child_result = self.wallet.create_self_transfer_multi( + utxos_to_spend=[parent_result["new_utxo"]], + num_outputs=num_child_outputs, + fee_per_output=int(child_fee * COIN // num_child_outputs), + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + package_hex = [parent_result["hex"], child_result["hex"]] + package_txns = [parent_result["tx"], child_result["tx"]] + return package_hex, package_txns + + def run_test(self): + # Counter used to count the number of times we constructed packages. Since we're constructing parent transactions with the same + # coins (to create conflicts), and perhaps giving them the same fee, we might accidentally just create the same transaction again. + # To prevent this, set nSequences to MAX_BIP125_RBF_SEQUENCE - self.ctr. + self.ctr = 0 + + self.log.info("Generate blocks to create UTXOs") + self.wallet = MiniWallet(self.nodes[0]) + + # Make more than enough coins for the sum of all tests, + # otherwise a wallet rescan is needed later + self.generate(self.wallet, 300) + self.coins = self.wallet.get_utxos(mark_as_spent=False) + + self.test_package_rbf_basic() + self.test_package_rbf_singleton() + self.test_package_rbf_additional_fees() + self.test_package_rbf_max_conflicts() + self.test_too_numerous_ancestors() + self.test_package_rbf_with_wrong_pkg_size() + self.test_insufficient_feerate() + self.test_wrong_conflict_cluster_size_linear() + self.test_wrong_conflict_cluster_size_parents_child() + self.test_wrong_conflict_cluster_size_parent_children() + self.test_0fee_package_rbf() + self.test_child_conflicts_parent_mempool_ancestor() + + def test_package_rbf_basic(self): + self.log.info("Test that a child can pay to replace its parents' conflicts of cluster size 2") + node = self.nodes[0] + # Reuse the same coins so that the transactions conflict with one another. + parent_coin = self.coins.pop() + package_hex1, package_txns1 = self.create_simple_package(parent_coin, DEFAULT_FEE, DEFAULT_FEE) + package_hex2, package_txns2 = self.create_simple_package(parent_coin, DEFAULT_FEE, DEFAULT_CHILD_FEE) + node.submitpackage(package_hex1) + self.assert_mempool_contents(expected=package_txns1) + + # Make sure 2nd node gets set up for basic package RBF + self.sync_all() + + # Test run rejected because conflicts are not allowed in subpackage evaluation + testres = node.testmempoolaccept(package_hex2) + assert_equal(testres[0]["reject-reason"], "bip125-replacement-disallowed") + + # But accepted during normal submission + submitres = node.submitpackage(package_hex2) + assert_equal(set(submitres["replaced-transactions"]), set([tx.rehash() for tx in package_txns1])) + self.assert_mempool_contents(expected=package_txns2) + + # Make sure 2nd node gets a basic package RBF over p2p + self.sync_all() + + self.generate(node, 1) + + def test_package_rbf_singleton(self): + self.log.info("Test child can pay to replace a parent's single conflicted tx") + node = self.nodes[0] + + # Make singleton tx to conflict with in next batch + singleton_coin = self.coins.pop() + singleton_tx = self.wallet.create_self_transfer(utxo_to_spend=singleton_coin) + node.sendrawtransaction(singleton_tx["hex"]) + self.assert_mempool_contents(expected=[singleton_tx["tx"]]) + + package_hex, package_txns = self.create_simple_package(singleton_coin, DEFAULT_FEE, singleton_tx["fee"] * 2) + + submitres = node.submitpackage(package_hex) + assert_equal(submitres["replaced-transactions"], [singleton_tx["tx"].rehash()]) + self.assert_mempool_contents(expected=package_txns) + + self.generate(node, 1) + + def test_package_rbf_additional_fees(self): + self.log.info("Check Package RBF must increase the absolute fee") + node = self.nodes[0] + coin = self.coins.pop() + + package_hex1, package_txns1 = self.create_simple_package(coin, parent_fee=DEFAULT_FEE, child_fee=DEFAULT_CHILD_FEE, heavy_child=True) + assert_greater_than_or_equal(1000, package_txns1[-1].get_vsize()) + node.submitpackage(package_hex1) + self.assert_mempool_contents(expected=package_txns1) + + PACKAGE_FEE = DEFAULT_FEE + DEFAULT_CHILD_FEE + PACKAGE_FEE_MINUS_ONE = PACKAGE_FEE - Decimal("0.00000001") + + # Package 2 has a higher feerate but lower absolute fee + package_hex2, package_txns2 = self.create_simple_package(coin, parent_fee=DEFAULT_FEE, child_fee=DEFAULT_CHILD_FEE - Decimal("0.00000001")) + pkg_results2 = node.submitpackage(package_hex2) + assert_equal(f"package RBF failed: insufficient anti-DoS fees, rejecting replacement {package_txns2[1].rehash()}, less fees than conflicting txs; {PACKAGE_FEE_MINUS_ONE} < {PACKAGE_FEE}", pkg_results2["package_msg"]) + self.assert_mempool_contents(expected=package_txns1) + + self.log.info("Check replacement pays for incremental bandwidth") + package_hex3, package_txns3 = self.create_simple_package(coin, parent_fee=DEFAULT_FEE, child_fee=DEFAULT_CHILD_FEE) + pkg_results3 = node.submitpackage(package_hex3) + assert_equal(f"package RBF failed: insufficient anti-DoS fees, rejecting replacement {package_txns3[1].rehash()}, not enough additional fees to relay; 0.00 < 0.00000{sum([tx.get_vsize() for tx in package_txns3])}", pkg_results3["package_msg"]) + + self.assert_mempool_contents(expected=package_txns1) + self.generate(node, 1) + + self.log.info("Check Package RBF must have strict cpfp structure") + coin = self.coins.pop() + package_hex4, package_txns4 = self.create_simple_package(coin, parent_fee=DEFAULT_FEE, child_fee=DEFAULT_CHILD_FEE) + node.submitpackage(package_hex4) + self.assert_mempool_contents(expected=package_txns4) + package_hex5, package_txns5 = self.create_simple_package(coin, parent_fee=DEFAULT_CHILD_FEE, child_fee=DEFAULT_CHILD_FEE - Decimal("0.00000001")) + pkg_results5 = node.submitpackage(package_hex5) + assert 'package RBF failed: package feerate is less than parent feerate' in pkg_results5["package_msg"] + + self.assert_mempool_contents(expected=package_txns4) + self.generate(node, 1) + + def test_package_rbf_max_conflicts(self): + node = self.nodes[0] + self.log.info("Check Package RBF cannot replace more than MAX_REPLACEMENT_CANDIDATES transactions") + num_coins = 51 + parent_coins = self.coins[:num_coins] + del self.coins[:num_coins] + + # Original transactions: 51 transactions with 1 descendants each -> 102 total transactions + size_two_clusters = [] + for coin in parent_coins: + size_two_clusters.append(self.wallet.send_self_transfer_chain(from_node=node, chain_length=2, utxo_to_spend=coin)) + expected_txns = [txn["tx"] for parent_child_txns in size_two_clusters for txn in parent_child_txns] + assert_equal(len(expected_txns), num_coins * 2) + self.assert_mempool_contents(expected=expected_txns) + + # parent feeerate needs to be high enough for minrelay + # child feerate needs to be large enough to trigger package rbf with a very large parent and + # pay for all evicted fees. maxfeerate turned off for all submissions since child feerate + # is extremely high + parent_fee_per_conflict = 10000 + child_feerate = 10000 * DEFAULT_FEE + + # Conflict against all transactions by double-spending each parent, causing 102 evictions + package_parent = self.wallet.create_self_transfer_multi(utxos_to_spend=parent_coins, fee_per_output=parent_fee_per_conflict) + package_child = self.wallet.create_self_transfer(fee_rate=child_feerate, utxo_to_spend=package_parent["new_utxos"][0]) + + pkg_results = node.submitpackage([package_parent["hex"], package_child["hex"]], maxfeerate=0) + assert_equal(f"package RBF failed: too many potential replacements, rejecting replacement {package_child['tx'].rehash()}; too many potential replacements (102 > 100)\n", pkg_results["package_msg"]) + self.assert_mempool_contents(expected=expected_txns) + + # Make singleton tx to conflict with in next batch + singleton_coin = self.coins.pop() + singleton_tx = self.wallet.create_self_transfer(utxo_to_spend=singleton_coin) + node.sendrawtransaction(singleton_tx["hex"]) + expected_txns.append(singleton_tx["tx"]) + + # Double-spend same set minus last, and double-spend singleton. This hits 101 evictions; should still fail. + # N.B. we can't RBF just a child tx in the clusters, as that would make resulting cluster of size 3. + double_spending_coins = parent_coins[:-1] + [singleton_coin] + package_parent = self.wallet.create_self_transfer_multi(utxos_to_spend=double_spending_coins, fee_per_output=parent_fee_per_conflict) + package_child = self.wallet.create_self_transfer(fee_rate=child_feerate, utxo_to_spend=package_parent["new_utxos"][0]) + pkg_results = node.submitpackage([package_parent["hex"], package_child["hex"]], maxfeerate=0) + assert_equal(f"package RBF failed: too many potential replacements, rejecting replacement {package_child['tx'].rehash()}; too many potential replacements (101 > 100)\n", pkg_results["package_msg"]) + self.assert_mempool_contents(expected=expected_txns) + + # Finally, evict MAX_REPLACEMENT_CANDIDATES + package_parent = self.wallet.create_self_transfer_multi(utxos_to_spend=parent_coins[:-1], fee_per_output=parent_fee_per_conflict) + package_child = self.wallet.create_self_transfer(fee_rate=child_feerate, utxo_to_spend=package_parent["new_utxos"][0]) + pkg_results = node.submitpackage([package_parent["hex"], package_child["hex"]], maxfeerate=0) + assert_equal(pkg_results["package_msg"], "success") + self.assert_mempool_contents(expected=[singleton_tx["tx"], size_two_clusters[-1][0]["tx"], size_two_clusters[-1][1]["tx"], package_parent["tx"], package_child["tx"]] ) + + self.generate(node, 1) + + def test_too_numerous_ancestors(self): + self.log.info("Test that package RBF doesn't work with packages larger than 2 due to ancestors") + node = self.nodes[0] + coin = self.coins.pop() + + package_hex1, package_txns1 = self.create_simple_package(coin, DEFAULT_FEE, DEFAULT_CHILD_FEE) + node.submitpackage(package_hex1) + self.assert_mempool_contents(expected=package_txns1) + + # Double-spends the original package + self.ctr += 1 + parent_result1 = self.wallet.create_self_transfer( + fee=DEFAULT_FEE, + utxo_to_spend=coin, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + coin2 = self.coins.pop() + + # Added to make package too large for package RBF; + # it will enter mempool individually + self.ctr += 1 + parent_result2 = self.wallet.create_self_transfer( + fee=DEFAULT_FEE, + utxo_to_spend=coin2, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + # Child that spends both, violating cluster size rule due + # to in-mempool ancestry + self.ctr += 1 + child_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_CHILD_FEE * COIN), + utxos_to_spend=[parent_result1["new_utxo"], parent_result2["new_utxo"]], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + package_hex2 = [parent_result1["hex"], parent_result2["hex"], child_result["hex"]] + package_txns2_succeed = [parent_result2["tx"]] + + pkg_result = node.submitpackage(package_hex2) + assert_equal(pkg_result["package_msg"], 'package RBF failed: new transaction cannot have mempool ancestors') + self.assert_mempool_contents(expected=package_txns1 + package_txns2_succeed) + self.generate(node, 1) + + def test_wrong_conflict_cluster_size_linear(self): + self.log.info("Test that conflicting with a cluster not sized two is rejected: linear chain") + node = self.nodes[0] + + # Coins we will conflict with + coin1 = self.coins.pop() + coin2 = self.coins.pop() + coin3 = self.coins.pop() + + # Three transactions chained; package RBF against any of these + # should be rejected + self.ctr += 1 + parent_result = self.wallet.create_self_transfer( + fee=DEFAULT_FEE, + utxo_to_spend=coin1, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + self.ctr += 1 + child_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * COIN), + utxos_to_spend=[parent_result["new_utxo"], coin2], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + self.ctr += 1 + grandchild_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * COIN), + utxos_to_spend=[child_result["new_utxos"][0], coin3], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + expected_txns = [parent_result["tx"], child_result["tx"], grandchild_result["tx"]] + for tx in expected_txns: + node.sendrawtransaction(tx.serialize().hex()) + self.assert_mempool_contents(expected=expected_txns) + + # Now make conflicting packages for each coin + package_hex1, package_txns1 = self.create_simple_package(coin1, DEFAULT_FEE, DEFAULT_CHILD_FEE) + + package_result = node.submitpackage(package_hex1) + assert_equal(f"package RBF failed: {parent_result['tx'].rehash()} has 2 descendants, max 1 allowed", package_result["package_msg"]) + + package_hex2, package_txns2 = self.create_simple_package(coin2, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex2) + assert_equal(f"package RBF failed: {child_result['tx'].rehash()} has both ancestor and descendant, exceeding cluster limit of 2", package_result["package_msg"]) + + package_hex3, package_txns3 = self.create_simple_package(coin3, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex3) + assert_equal(f"package RBF failed: {grandchild_result['tx'].rehash()} has 2 ancestors, max 1 allowed", package_result["package_msg"]) + + # Check that replacements were actually rejected + self.assert_mempool_contents(expected=expected_txns) + self.generate(node, 1) + + def test_wrong_conflict_cluster_size_parents_child(self): + self.log.info("Test that conflicting with a cluster not sized two is rejected: two parents one child") + node = self.nodes[0] + + # Coins we will conflict with + coin1 = self.coins.pop() + coin2 = self.coins.pop() + coin3 = self.coins.pop() + + self.ctr += 1 + parent1_result = self.wallet.create_self_transfer( + fee=DEFAULT_FEE, + utxo_to_spend=coin1, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + self.ctr += 1 + parent2_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * COIN), + utxos_to_spend=[coin2], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + self.ctr += 1 + child_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * COIN), + utxos_to_spend=[parent1_result["new_utxo"], parent2_result["new_utxos"][0], coin3], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + expected_txns = [parent1_result["tx"], parent2_result["tx"], child_result["tx"]] + for tx in expected_txns: + node.sendrawtransaction(tx.serialize().hex()) + self.assert_mempool_contents(expected=expected_txns) + + # Now make conflicting packages for each coin + package_hex1, package_txns1 = self.create_simple_package(coin1, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex1) + assert_equal(f"package RBF failed: {parent1_result['tx'].rehash()} is not the only parent of child {child_result['tx'].rehash()}", package_result["package_msg"]) + + package_hex2, package_txns2 = self.create_simple_package(coin2, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex2) + assert_equal(f"package RBF failed: {parent2_result['tx'].rehash()} is not the only parent of child {child_result['tx'].rehash()}", package_result["package_msg"]) + + package_hex3, package_txns3 = self.create_simple_package(coin3, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex3) + assert_equal(f"package RBF failed: {child_result['tx'].rehash()} has 2 ancestors, max 1 allowed", package_result["package_msg"]) + + # Check that replacements were actually rejected + self.assert_mempool_contents(expected=expected_txns) + self.generate(node, 1) + + def test_wrong_conflict_cluster_size_parent_children(self): + self.log.info("Test that conflicting with a cluster not sized two is rejected: one parent two children") + node = self.nodes[0] + + # Coins we will conflict with + coin1 = self.coins.pop() + coin2 = self.coins.pop() + coin3 = self.coins.pop() + + self.ctr += 1 + parent_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * COIN), + num_outputs=2, + utxos_to_spend=[coin1], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + self.ctr += 1 + child1_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * COIN), + utxos_to_spend=[parent_result["new_utxos"][0], coin2], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + self.ctr += 1 + child2_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * COIN), + utxos_to_spend=[parent_result["new_utxos"][1], coin3], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + # Submit them to mempool + expected_txns = [parent_result["tx"], child1_result["tx"], child2_result["tx"]] + for tx in expected_txns: + node.sendrawtransaction(tx.serialize().hex()) + self.assert_mempool_contents(expected=expected_txns) + + # Now make conflicting packages for each coin + package_hex1, package_txns1 = self.create_simple_package(coin1, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex1) + assert_equal(f"package RBF failed: {parent_result['tx'].rehash()} has 2 descendants, max 1 allowed", package_result["package_msg"]) + + package_hex2, package_txns2 = self.create_simple_package(coin2, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex2) + assert_equal(f"package RBF failed: {child1_result['tx'].rehash()} is not the only child of parent {parent_result['tx'].rehash()}", package_result["package_msg"]) + + package_hex3, package_txns3 = self.create_simple_package(coin3, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex3) + assert_equal(f"package RBF failed: {child2_result['tx'].rehash()} is not the only child of parent {parent_result['tx'].rehash()}", package_result["package_msg"]) + + # Check that replacements were actually rejected + self.assert_mempool_contents(expected=expected_txns) + self.generate(node, 1) + + def test_package_rbf_with_wrong_pkg_size(self): + self.log.info("Test that package RBF doesn't work with packages larger than 2 due to pkg size") + node = self.nodes[0] + coin1 = self.coins.pop() + coin2 = self.coins.pop() + + # Two packages to require multiple direct conflicts, easier to set up illicit pkg size + package_hex1, package_txns1 = self.create_simple_package(coin1, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_hex2, package_txns2 = self.create_simple_package(coin2, DEFAULT_FEE, DEFAULT_CHILD_FEE) + + node.submitpackage(package_hex1) + node.submitpackage(package_hex2) + + self.assert_mempool_contents(expected=package_txns1 + package_txns2) + assert_equal(len(node.getrawmempool()), 4) + + # Double-spends the first package + self.ctr += 1 + parent_result1 = self.wallet.create_self_transfer( + fee=DEFAULT_FEE, + utxo_to_spend=coin1, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + # Double-spends the second package + self.ctr += 1 + parent_result2 = self.wallet.create_self_transfer( + fee=DEFAULT_FEE, + utxo_to_spend=coin2, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + # Child that spends both, violating cluster size rule due + # to pkg size + self.ctr += 1 + child_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_CHILD_FEE * COIN), + utxos_to_spend=[parent_result1["new_utxo"], parent_result2["new_utxo"]], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + package_hex3 = [parent_result1["hex"], parent_result2["hex"], child_result["hex"]] + + pkg_result = node.submitpackage(package_hex3) + assert_equal(pkg_result["package_msg"], 'package RBF failed: package must be 1-parent-1-child') + self.assert_mempool_contents(expected=package_txns1 + package_txns2) + self.generate(node, 1) + + def test_insufficient_feerate(self): + self.log.info("Check Package RBF must beat feerate of direct conflict") + node = self.nodes[0] + coin = self.coins.pop() + + # Non-cpfp structure + package_hex1, package_txns1 = self.create_simple_package(coin, parent_fee=DEFAULT_CHILD_FEE, child_fee=DEFAULT_FEE) + node.submitpackage(package_hex1) + self.assert_mempool_contents(expected=package_txns1) + + # Package 2 feerate is below the feerate of directly conflicted parent, so it fails even though + # total fees are higher than the original package + package_hex2, package_txns2 = self.create_simple_package(coin, parent_fee=DEFAULT_CHILD_FEE - Decimal("0.00000001"), child_fee=DEFAULT_CHILD_FEE) + pkg_results2 = node.submitpackage(package_hex2) + assert_equal(pkg_results2["package_msg"], 'package RBF failed: insufficient feerate: does not improve feerate diagram') + self.assert_mempool_contents(expected=package_txns1) + self.generate(node, 1) + + def test_0fee_package_rbf(self): + self.log.info("Test package RBF: TRUC 0-fee parent + high-fee child replaces parent's conflicts") + node = self.nodes[0] + # Reuse the same coins so that the transactions conflict with one another. + self.wallet.rescan_utxos() + parent_coin = self.wallet.get_utxo(confirmed_only=True) + + # package1 pays default fee on both transactions + parent1 = self.wallet.create_self_transfer(utxo_to_spend=parent_coin, version=3) + child1 = self.wallet.create_self_transfer(utxo_to_spend=parent1["new_utxo"], version=3) + package_hex1 = [parent1["hex"], child1["hex"]] + fees_package1 = parent1["fee"] + child1["fee"] + submitres1 = node.submitpackage(package_hex1) + assert_equal(submitres1["package_msg"], "success") + self.assert_mempool_contents([parent1["tx"], child1["tx"]]) + + # package2 has a 0-fee parent (conflicting with package1) and very high fee child + parent2 = self.wallet.create_self_transfer(utxo_to_spend=parent_coin, fee=0, fee_rate=0, version=3) + child2 = self.wallet.create_self_transfer(utxo_to_spend=parent2["new_utxo"], fee=fees_package1*10, version=3) + package_hex2 = [parent2["hex"], child2["hex"]] + + submitres2 = node.submitpackage(package_hex2) + assert_equal(submitres2["package_msg"], "success") + assert_equal(set(submitres2["replaced-transactions"]), set([parent1["txid"], child1["txid"]])) + self.assert_mempool_contents([parent2["tx"], child2["tx"]]) + + self.generate(node, 1) + + def test_child_conflicts_parent_mempool_ancestor(self): + fill_mempool(self, self.nodes[0]) + # Reset coins since we filled the mempool with current coins + self.coins = self.wallet.get_utxos(mark_as_spent=False, confirmed_only=True) + + self.log.info("Test that package RBF doesn't have issues with mempool<->package conflicts via inconsistency") + node = self.nodes[0] + coin = self.coins.pop() + + self.ctr += 1 + grandparent_result = self.wallet.create_self_transfer( + fee=DEFAULT_FEE, + utxo_to_spend=coin, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + node.sendrawtransaction(grandparent_result["hex"]) + + # Now make package of two descendants that looks + # like a cpfp where the parent can't get in on its own + self.ctr += 1 + parent_result = self.wallet.create_self_transfer( + fee_rate=Decimal('0.00001000'), + utxo_to_spend=grandparent_result["new_utxo"], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + # Last tx double-spends grandparent's coin, + # which is not inside the current package + self.ctr += 1 + child_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_CHILD_FEE * COIN), + utxos_to_spend=[parent_result["new_utxo"], coin], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + pkg_result = node.submitpackage([parent_result["hex"], child_result["hex"]]) + assert_equal(pkg_result["package_msg"], 'package RBF failed: new transaction cannot have mempool ancestors') + mempool_info = node.getrawmempool() + assert grandparent_result["txid"] in mempool_info + assert parent_result["txid"] not in mempool_info + assert child_result["txid"] not in mempool_info + +if __name__ == "__main__": + PackageRBFTest().main() diff --git a/test/functional/p2p_disconnect_ban.py b/test/functional/p2p_disconnect_ban.py index 678b006886..e47f9c732b 100755 --- a/test/functional/p2p_disconnect_ban.py +++ b/test/functional/p2p_disconnect_ban.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2022 The Bitcoin Core developers +# Copyright (c) 2014-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node disconnect and ban behavior""" @@ -18,7 +18,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.supports_cli = False def run_test(self): - self.log.info("Connect nodes both way") + self.log.info("Connect nodes both ways") # By default, the test framework sets up an addnode connection from # node 1 --> node0. By connecting node0 --> node 1, we're left with # the two nodes being connected both ways. @@ -84,7 +84,7 @@ class DisconnectBanTest(BitcoinTestFramework): assert_equal("192.168.0.1/32", listBeforeShutdown[2]['address']) self.log.info("setban: test banning with absolute timestamp") - self.nodes[1].setban("192.168.0.2", "add", old_time + 120, True) + self.nodes[1].setban("192.168.0.2", "add", old_time + 120, absolute=True) # Move time forward by 3 seconds so the fourth ban has expired self.nodes[1].setmocktime(old_time + 3) @@ -102,7 +102,9 @@ class DisconnectBanTest(BitcoinTestFramework): assert_equal(ban["ban_duration"], 120) assert_equal(ban["time_remaining"], 117) - self.restart_node(1) + # Keep mocktime, to avoid ban expiry when restart takes longer than + # time_remaining + self.restart_node(1, extra_args=[f"-mocktime={old_time+4}"]) listAfterShutdown = self.nodes[1].listbanned() assert_equal("127.0.0.0/24", listAfterShutdown[0]['address']) @@ -113,7 +115,7 @@ class DisconnectBanTest(BitcoinTestFramework): # Clear ban lists self.nodes[1].clearbanned() - self.log.info("Connect nodes both way") + self.log.info("Connect nodes both ways") self.connect_nodes(0, 1) self.connect_nodes(1, 0) diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index cb78313786..8e459ba676 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -5,7 +5,6 @@ """Test node responses to invalid network messages.""" import random -import struct import time from test_framework.messages import ( @@ -233,7 +232,7 @@ class InvalidMessagesTest(BitcoinTestFramework): '208d')) # port def test_addrv2_unrecognized_network(self): - now_hex = struct.pack('<I', int(time.time())).hex() + now_hex = int(time.time()).to_bytes(4, "little").hex() self.test_addrv2('unrecognized network', [ 'received: addrv2 (25 bytes)', diff --git a/test/functional/p2p_mutated_blocks.py b/test/functional/p2p_mutated_blocks.py index 6a9bbc0a8c..708b19b1e5 100755 --- a/test/functional/p2p_mutated_blocks.py +++ b/test/functional/p2p_mutated_blocks.py @@ -55,7 +55,7 @@ class MutatedBlocksTest(BitcoinTestFramework): # Create mutated version of the block by changing the transaction # version on the self-transfer. mutated_block = copy.deepcopy(block) - mutated_block.vtx[1].nVersion = 4 + mutated_block.vtx[1].version = 4 # Announce the new block via a compact block through the honest relayer cmpctblock = HeaderAndShortIDs() diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 45bbd7f1c3..d20cf41a72 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -5,7 +5,6 @@ """Test segwit transactions and blocks on P2P network.""" from decimal import Decimal import random -import struct import time from test_framework.blocktools import ( @@ -1165,16 +1164,16 @@ class SegWitTest(BitcoinTestFramework): if not self.wit.is_null(): flags |= 1 r = b"" - r += struct.pack("<i", self.nVersion) + r += self.version.to_bytes(4, "little") if flags: dummy = [] r += ser_vector(dummy) - r += struct.pack("<B", flags) + r += flags.to_bytes(1, "little") r += ser_vector(self.vin) r += ser_vector(self.vout) if flags & 1: r += self.wit.serialize() - r += struct.pack("<I", self.nLockTime) + r += self.nLockTime.to_bytes(4, "little") return r tx2 = BrokenCTransaction() @@ -1976,11 +1975,11 @@ class SegWitTest(BitcoinTestFramework): def serialize_with_bogus_witness(tx): flags = 3 r = b"" - r += struct.pack("<i", tx.nVersion) + r += tx.version.to_bytes(4, "little") if flags: dummy = [] r += ser_vector(dummy) - r += struct.pack("<B", flags) + r += flags.to_bytes(1, "little") r += ser_vector(tx.vin) r += ser_vector(tx.vout) if flags & 1: @@ -1990,7 +1989,7 @@ class SegWitTest(BitcoinTestFramework): for _ in range(len(tx.wit.vtxinwit), len(tx.vin)): tx.wit.vtxinwit.append(CTxInWitness()) r += tx.wit.serialize() - r += struct.pack("<I", tx.nLockTime) + r += tx.nLockTime.to_bytes(4, "little") return r class msg_bogus_tx(msg_tx): diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index 65d7b4c422..37656341d2 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -9,10 +9,10 @@ import json import os from test_framework.address import address_to_scriptpubkey -from test_framework.blocktools import COINBASE_MATURITY -from test_framework.authproxy import JSONRPCException from test_framework.descriptors import descsum_create, drop_origins from test_framework.key import ECPubKey +from test_framework.messages import COIN +from test_framework.script_util import keys_to_multisig_script from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_raises_rpc_error, @@ -32,92 +32,54 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 3 self.supports_cli = False + self.enable_wallet_if_possible() - def get_keys(self): + def create_keys(self, num_keys): self.pub = [] self.priv = [] - node0, node1, node2 = self.nodes - for _ in range(self.nkeys): + for _ in range(num_keys): privkey, pubkey = generate_keypair(wif=True) self.pub.append(pubkey.hex()) self.priv.append(privkey) - if self.is_bdb_compiled(): - self.final = node2.getnewaddress() - else: - self.final = getnewdestination('bech32')[2] + + def create_wallet(self, node, wallet_name): + node.createwallet(wallet_name=wallet_name, disable_private_keys=True) + return node.get_wallet_rpc(wallet_name) def run_test(self): node0, node1, node2 = self.nodes self.wallet = MiniWallet(test_node=node0) - if self.is_bdb_compiled(): - self.import_deterministic_coinbase_privkeys() + if self.is_wallet_compiled(): self.check_addmultisigaddress_errors() self.log.info('Generating blocks ...') self.generate(self.wallet, 149) - self.moved = 0 - for self.nkeys in [3, 5]: - for self.nsigs in [2, 3]: - for self.output_type in ["bech32", "p2sh-segwit", "legacy"]: - self.get_keys() - self.do_multisig() - if self.is_bdb_compiled(): - self.checkbalances() - - # Test mixed compressed and uncompressed pubkeys - self.log.info('Mixed compressed and uncompressed multisigs are not allowed') - pk0, pk1, pk2 = [getnewdestination('bech32')[0].hex() for _ in range(3)] - - # decompress pk2 - pk_obj = ECPubKey() - pk_obj.set(bytes.fromhex(pk2)) - pk_obj.compressed = False - pk2 = pk_obj.get_bytes().hex() - - if self.is_bdb_compiled(): - node0.createwallet(wallet_name='wmulti0', disable_private_keys=True) - wmulti0 = node0.get_wallet_rpc('wmulti0') - - # Check all permutations of keys because order matters apparently - for keys in itertools.permutations([pk0, pk1, pk2]): - # Results should be the same as this legacy one - legacy_addr = node0.createmultisig(2, keys, 'legacy')['address'] - - if self.is_bdb_compiled(): - result = wmulti0.addmultisigaddress(2, keys, '', 'legacy') - assert_equal(legacy_addr, result['address']) - assert 'warnings' not in result - - # Generate addresses with the segwit types. These should all make legacy addresses - err_msg = ["Unable to make chosen address type, please ensure no uncompressed public keys are present."] - - for addr_type in ['bech32', 'p2sh-segwit']: - result = self.nodes[0].createmultisig(nrequired=2, keys=keys, address_type=addr_type) - assert_equal(legacy_addr, result['address']) - assert_equal(result['warnings'], err_msg) - - if self.is_bdb_compiled(): - result = wmulti0.addmultisigaddress(nrequired=2, keys=keys, address_type=addr_type) - assert_equal(legacy_addr, result['address']) - assert_equal(result['warnings'], err_msg) - - self.log.info('Testing sortedmulti descriptors with BIP 67 test vectors') - with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/rpc_bip67.json'), encoding='utf-8') as f: - vectors = json.load(f) + wallet_multi = self.create_wallet(node1, 'wmulti') if self._requires_wallet else None + self.create_keys(21) # max number of allowed keys + 1 + m_of_n = [(2, 3), (3, 3), (2, 5), (3, 5), (10, 15), (15, 15)] + for (sigs, keys) in m_of_n: + for output_type in ["bech32", "p2sh-segwit", "legacy"]: + self.do_multisig(keys, sigs, output_type, wallet_multi) - for t in vectors: - key_str = ','.join(t['keys']) - desc = descsum_create('sh(sortedmulti(2,{}))'.format(key_str)) - assert_equal(self.nodes[0].deriveaddresses(desc)[0], t['address']) - sorted_key_str = ','.join(t['sorted_keys']) - sorted_key_desc = descsum_create('sh(multi(2,{}))'.format(sorted_key_str)) - assert_equal(self.nodes[0].deriveaddresses(sorted_key_desc)[0], t['address']) + self.test_multisig_script_limit(wallet_multi) + self.test_mixing_uncompressed_and_compressed_keys(node0, wallet_multi) + self.test_sortedmulti_descriptors_bip67() # Check that bech32m is currently not allowed assert_raises_rpc_error(-5, "createmultisig cannot create bech32m multisig addresses", self.nodes[0].createmultisig, 2, self.pub, "bech32m") + self.log.info('Check correct encoding of multisig script for all n (1..20)') + for nkeys in range(1, 20+1): + keys = [self.pub[0]]*nkeys + expected_ms_script = keys_to_multisig_script(keys, k=nkeys) # simply use n-of-n + # note that the 'legacy' address type fails for n values larger than 15 + # due to exceeding the P2SH size limit (520 bytes), so we use 'bech32' instead + # (for the purpose of this encoding test, we don't care about the resulting address) + res = self.nodes[0].createmultisig(nrequired=nkeys, keys=keys, address_type='bech32') + assert_equal(res['redeemScript'], expected_ms_script.hex()) + def check_addmultisigaddress_errors(self): if self.options.descriptors: return @@ -133,117 +95,165 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): pubs = [self.nodes[1].getaddressinfo(addr)["pubkey"] for addr in addresses] assert_raises_rpc_error(-5, "Bech32m multisig addresses cannot be created with legacy wallets", self.nodes[0].addmultisigaddress, 2, pubs, "", "bech32m") - def checkbalances(self): - node0, node1, node2 = self.nodes - self.generate(node0, COINBASE_MATURITY) + def test_multisig_script_limit(self, wallet_multi): + node1 = self.nodes[1] + pubkeys = self.pub[0:20] - bal0 = node0.getbalance() - bal1 = node1.getbalance() - bal2 = node2.getbalance() - balw = self.wallet.get_balance() + self.log.info('Test legacy redeem script max size limit') + assert_raises_rpc_error(-8, "redeemScript exceeds size limit: 684 > 520", node1.createmultisig, 16, pubkeys, 'legacy') - height = node0.getblockchaininfo()["blocks"] - assert 150 < height < 350 - total = 149 * 50 + (height - 149 - 100) * 25 - assert bal1 == 0 - assert bal2 == self.moved - assert_equal(bal0 + bal1 + bal2 + balw, total) + self.log.info('Test valid 16-20 multisig p2sh-legacy and bech32 (no wallet)') + self.do_multisig(nkeys=20, nsigs=16, output_type="p2sh-segwit", wallet_multi=None) + self.do_multisig(nkeys=20, nsigs=16, output_type="bech32", wallet_multi=None) - def do_multisig(self): - node0, node1, node2 = self.nodes + self.log.info('Test invalid 16-21 multisig p2sh-legacy and bech32 (no wallet)') + assert_raises_rpc_error(-8, "Number of keys involved in the multisignature address creation > 20", node1.createmultisig, 16, self.pub, 'p2sh-segwit') + assert_raises_rpc_error(-8, "Number of keys involved in the multisignature address creation > 20", node1.createmultisig, 16, self.pub, 'bech32') - if self.is_bdb_compiled(): - if 'wmulti' not in node1.listwallets(): - try: - node1.loadwallet('wmulti') - except JSONRPCException as e: - path = self.nodes[1].wallets_path / "wmulti" - if e.error['code'] == -18 and "Wallet file verification failed. Failed to load database path '{}'. Path does not exist.".format(path) in e.error['message']: - node1.createwallet(wallet_name='wmulti', disable_private_keys=True) - else: - raise - wmulti = node1.get_wallet_rpc('wmulti') + # Check legacy wallet related command + self.log.info('Test legacy redeem script max size limit (with wallet)') + if wallet_multi is not None and not self.options.descriptors: + assert_raises_rpc_error(-8, "redeemScript exceeds size limit: 684 > 520", wallet_multi.addmultisigaddress, 16, pubkeys, '', 'legacy') + + self.log.info('Test legacy wallet unsupported operation. 16-20 multisig p2sh-legacy and bech32 generation') + # Due an internal limitation on legacy wallets, the redeem script limit also applies to p2sh-segwit and bech32 (even when the scripts are valid) + # We take this as a "good thing" to tell users to upgrade to descriptors. + assert_raises_rpc_error(-4, "Unsupported multisig script size for legacy wallet. Upgrade to descriptors to overcome this limitation for p2sh-segwit or bech32 scripts", wallet_multi.addmultisigaddress, 16, pubkeys, '', 'p2sh-segwit') + assert_raises_rpc_error(-4, "Unsupported multisig script size for legacy wallet. Upgrade to descriptors to overcome this limitation for p2sh-segwit or bech32 scripts", wallet_multi.addmultisigaddress, 16, pubkeys, '', 'bech32') + + def do_multisig(self, nkeys, nsigs, output_type, wallet_multi): + node0, node1, node2 = self.nodes + pub_keys = self.pub[0: nkeys] + priv_keys = self.priv[0: nkeys] # Construct the expected descriptor - desc = 'multi({},{})'.format(self.nsigs, ','.join(self.pub)) - if self.output_type == 'legacy': + desc = 'multi({},{})'.format(nsigs, ','.join(pub_keys)) + if output_type == 'legacy': desc = 'sh({})'.format(desc) - elif self.output_type == 'p2sh-segwit': + elif output_type == 'p2sh-segwit': desc = 'sh(wsh({}))'.format(desc) - elif self.output_type == 'bech32': + elif output_type == 'bech32': desc = 'wsh({})'.format(desc) desc = descsum_create(desc) - msig = node2.createmultisig(self.nsigs, self.pub, self.output_type) + msig = node2.createmultisig(nsigs, pub_keys, output_type) assert 'warnings' not in msig madd = msig["address"] mredeem = msig["redeemScript"] assert_equal(desc, msig['descriptor']) - if self.output_type == 'bech32': + if output_type == 'bech32': assert madd[0:4] == "bcrt" # actually a bech32 address - if self.is_bdb_compiled(): + if wallet_multi is not None: # compare against addmultisigaddress - msigw = wmulti.addmultisigaddress(self.nsigs, self.pub, None, self.output_type) + msigw = wallet_multi.addmultisigaddress(nsigs, pub_keys, None, output_type) maddw = msigw["address"] mredeemw = msigw["redeemScript"] assert_equal(desc, drop_origins(msigw['descriptor'])) # addmultisigiaddress and createmultisig work the same assert maddw == madd assert mredeemw == mredeem - wmulti.unloadwallet() spk = address_to_scriptpubkey(madd) - txid = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=spk, amount=1300)["txid"] - tx = node0.getrawtransaction(txid, True) - vout = [v["n"] for v in tx["vout"] if madd == v["scriptPubKey"]["address"]] - assert len(vout) == 1 - vout = vout[0] - scriptPubKey = tx["vout"][vout]["scriptPubKey"]["hex"] - value = tx["vout"][vout]["value"] - prevtxs = [{"txid": txid, "vout": vout, "scriptPubKey": scriptPubKey, "redeemScript": mredeem, "amount": value}] + value = decimal.Decimal("0.00004000") + tx = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=spk, amount=int(value * COIN)) + prevtxs = [{"txid": tx["txid"], "vout": tx["sent_vout"], "scriptPubKey": spk.hex(), "redeemScript": mredeem, "amount": value}] self.generate(node0, 1) - outval = value - decimal.Decimal("0.00001000") - rawtx = node2.createrawtransaction([{"txid": txid, "vout": vout}], [{self.final: outval}]) + outval = value - decimal.Decimal("0.00002000") # deduce fee (must be higher than the min relay fee) + # send coins to node2 when wallet is enabled + node2_balance = node2.getbalances()['mine']['trusted'] if self.is_wallet_compiled() else 0 + out_addr = node2.getnewaddress() if self.is_wallet_compiled() else getnewdestination('bech32')[2] + rawtx = node2.createrawtransaction([{"txid": tx["txid"], "vout": tx["sent_vout"]}], [{out_addr: outval}]) prevtx_err = dict(prevtxs[0]) del prevtx_err["redeemScript"] - assert_raises_rpc_error(-8, "Missing redeemScript/witnessScript", node2.signrawtransactionwithkey, rawtx, self.priv[0:self.nsigs-1], [prevtx_err]) + assert_raises_rpc_error(-8, "Missing redeemScript/witnessScript", node2.signrawtransactionwithkey, rawtx, priv_keys[0:nsigs-1], [prevtx_err]) # if witnessScript specified, all ok prevtx_err["witnessScript"] = prevtxs[0]["redeemScript"] - node2.signrawtransactionwithkey(rawtx, self.priv[0:self.nsigs-1], [prevtx_err]) + node2.signrawtransactionwithkey(rawtx, priv_keys[0:nsigs-1], [prevtx_err]) # both specified, also ok prevtx_err["redeemScript"] = prevtxs[0]["redeemScript"] - node2.signrawtransactionwithkey(rawtx, self.priv[0:self.nsigs-1], [prevtx_err]) + node2.signrawtransactionwithkey(rawtx, priv_keys[0:nsigs-1], [prevtx_err]) # redeemScript mismatch to witnessScript prevtx_err["redeemScript"] = "6a" # OP_RETURN - assert_raises_rpc_error(-8, "redeemScript does not correspond to witnessScript", node2.signrawtransactionwithkey, rawtx, self.priv[0:self.nsigs-1], [prevtx_err]) + assert_raises_rpc_error(-8, "redeemScript does not correspond to witnessScript", node2.signrawtransactionwithkey, rawtx, priv_keys[0:nsigs-1], [prevtx_err]) # redeemScript does not match scriptPubKey del prevtx_err["witnessScript"] - assert_raises_rpc_error(-8, "redeemScript/witnessScript does not match scriptPubKey", node2.signrawtransactionwithkey, rawtx, self.priv[0:self.nsigs-1], [prevtx_err]) + assert_raises_rpc_error(-8, "redeemScript/witnessScript does not match scriptPubKey", node2.signrawtransactionwithkey, rawtx, priv_keys[0:nsigs-1], [prevtx_err]) # witnessScript does not match scriptPubKey prevtx_err["witnessScript"] = prevtx_err["redeemScript"] del prevtx_err["redeemScript"] - assert_raises_rpc_error(-8, "redeemScript/witnessScript does not match scriptPubKey", node2.signrawtransactionwithkey, rawtx, self.priv[0:self.nsigs-1], [prevtx_err]) + assert_raises_rpc_error(-8, "redeemScript/witnessScript does not match scriptPubKey", node2.signrawtransactionwithkey, rawtx, priv_keys[0:nsigs-1], [prevtx_err]) - rawtx2 = node2.signrawtransactionwithkey(rawtx, self.priv[0:self.nsigs - 1], prevtxs) - rawtx3 = node2.signrawtransactionwithkey(rawtx2["hex"], [self.priv[-1]], prevtxs) + rawtx2 = node2.signrawtransactionwithkey(rawtx, priv_keys[0:nsigs - 1], prevtxs) + rawtx3 = node2.signrawtransactionwithkey(rawtx2["hex"], [priv_keys[-1]], prevtxs) + assert rawtx3['complete'] - self.moved += outval tx = node0.sendrawtransaction(rawtx3["hex"], 0) blk = self.generate(node0, 1)[0] assert tx in node0.getblock(blk)["tx"] + # When the wallet is enabled, assert node2 sees the incoming amount + if self.is_wallet_compiled(): + assert_equal(node2.getbalances()['mine']['trusted'], node2_balance + outval) + txinfo = node0.getrawtransaction(tx, True, blk) - self.log.info("n/m=%d/%d %s size=%d vsize=%d weight=%d" % (self.nsigs, self.nkeys, self.output_type, txinfo["size"], txinfo["vsize"], txinfo["weight"])) + self.log.info("n/m=%d/%d %s size=%d vsize=%d weight=%d" % (nsigs, nkeys, output_type, txinfo["size"], txinfo["vsize"], txinfo["weight"])) + + def test_mixing_uncompressed_and_compressed_keys(self, node, wallet_multi): + self.log.info('Mixed compressed and uncompressed multisigs are not allowed') + pk0, pk1, pk2 = [getnewdestination('bech32')[0].hex() for _ in range(3)] + + # decompress pk2 + pk_obj = ECPubKey() + pk_obj.set(bytes.fromhex(pk2)) + pk_obj.compressed = False + pk2 = pk_obj.get_bytes().hex() + + # Check all permutations of keys because order matters apparently + for keys in itertools.permutations([pk0, pk1, pk2]): + # Results should be the same as this legacy one + legacy_addr = node.createmultisig(2, keys, 'legacy')['address'] + + if wallet_multi is not None: + # 'addmultisigaddress' should return the same address + result = wallet_multi.addmultisigaddress(2, keys, '', 'legacy') + assert_equal(legacy_addr, result['address']) + assert 'warnings' not in result + + # Generate addresses with the segwit types. These should all make legacy addresses + err_msg = ["Unable to make chosen address type, please ensure no uncompressed public keys are present."] + + for addr_type in ['bech32', 'p2sh-segwit']: + result = self.nodes[0].createmultisig(nrequired=2, keys=keys, address_type=addr_type) + assert_equal(legacy_addr, result['address']) + assert_equal(result['warnings'], err_msg) + + if wallet_multi is not None: + result = wallet_multi.addmultisigaddress(nrequired=2, keys=keys, address_type=addr_type) + assert_equal(legacy_addr, result['address']) + assert_equal(result['warnings'], err_msg) + + def test_sortedmulti_descriptors_bip67(self): + self.log.info('Testing sortedmulti descriptors with BIP 67 test vectors') + with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/rpc_bip67.json'), encoding='utf-8') as f: + vectors = json.load(f) + + for t in vectors: + key_str = ','.join(t['keys']) + desc = descsum_create('sh(sortedmulti(2,{}))'.format(key_str)) + assert_equal(self.nodes[0].deriveaddresses(desc)[0], t['address']) + sorted_key_str = ','.join(t['sorted_keys']) + sorted_key_desc = descsum_create('sh(multi(2,{}))'.format(sorted_key_str)) + assert_equal(self.nodes[0].deriveaddresses(sorted_key_desc)[0], t['address']) if __name__ == '__main__': diff --git a/test/functional/rpc_packages.py b/test/functional/rpc_packages.py index 113424c0a6..1acd586d2c 100755 --- a/test/functional/rpc_packages.py +++ b/test/functional/rpc_packages.py @@ -394,7 +394,7 @@ class RPCPackagesTest(BitcoinTestFramework): peer = node.add_p2p_connection(P2PTxInvStore()) txs = self.wallet.create_self_transfer_chain(chain_length=2) bad_child = tx_from_hex(txs[1]["hex"]) - bad_child.nVersion = -1 + bad_child.version = 0xffffffff hex_partial_acceptance = [txs[0]["hex"], bad_child.serialize().hex()] res = node.submitpackage(hex_partial_acceptance) assert_equal(res["package_msg"], "transaction failed") diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index 3978c80dde..f974a05f7b 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -463,20 +463,34 @@ class RawTransactionsTest(BitcoinTestFramework): self.log.info("Test transaction version numbers") # Test the minimum transaction version number that fits in a signed 32-bit integer. - # As transaction version is unsigned, this should convert to its unsigned equivalent. + # As transaction version is serialized unsigned, this should convert to its unsigned equivalent. tx = CTransaction() - tx.nVersion = -0x80000000 + tx.version = 0x80000000 rawtx = tx.serialize().hex() decrawtx = self.nodes[0].decoderawtransaction(rawtx) assert_equal(decrawtx['version'], 0x80000000) # Test the maximum transaction version number that fits in a signed 32-bit integer. tx = CTransaction() - tx.nVersion = 0x7fffffff + tx.version = 0x7fffffff rawtx = tx.serialize().hex() decrawtx = self.nodes[0].decoderawtransaction(rawtx) assert_equal(decrawtx['version'], 0x7fffffff) + # Test the minimum transaction version number that fits in an unsigned 32-bit integer. + tx = CTransaction() + tx.version = 0 + rawtx = tx.serialize().hex() + decrawtx = self.nodes[0].decoderawtransaction(rawtx) + assert_equal(decrawtx['version'], 0) + + # Test the maximum transaction version number that fits in an unsigned 32-bit integer. + tx = CTransaction() + tx.version = 0xffffffff + rawtx = tx.serialize().hex() + decrawtx = self.nodes[0].decoderawtransaction(rawtx) + assert_equal(decrawtx['version'], 0xffffffff) + def raw_multisig_transaction_legacy_tests(self): self.log.info("Test raw multisig transactions (legacy)") # The traditional multisig workflow does not work with descriptor wallets so these are legacy only. @@ -585,6 +599,8 @@ class RawTransactionsTest(BitcoinTestFramework): rawTxPartialSigned2 = self.nodes[2].signrawtransactionwithwallet(rawTx2, inputs) self.log.debug(rawTxPartialSigned2) assert_equal(rawTxPartialSigned2['complete'], False) # node2 only has one key, can't comp. sign the tx + assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].combinerawtransaction, [rawTxPartialSigned1['hex'], rawTxPartialSigned2['hex'] + "00"]) + assert_raises_rpc_error(-22, "Missing transactions", self.nodes[0].combinerawtransaction, []) rawTxComb = self.nodes[2].combinerawtransaction([rawTxPartialSigned1['hex'], rawTxPartialSigned2['hex']]) self.log.debug(rawTxComb) self.nodes[2].sendrawtransaction(rawTxComb) @@ -592,6 +608,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.sync_all() self.generate(self.nodes[0], 1) assert_equal(self.nodes[0].getbalance(), bal + Decimal('50.00000000') + Decimal('2.19000000')) # block reward + tx + assert_raises_rpc_error(-25, "Input not found or already spent", self.nodes[0].combinerawtransaction, [rawTxPartialSigned1['hex'], rawTxPartialSigned2['hex']]) if __name__ == '__main__': diff --git a/test/functional/rpc_signrawtransactionwithkey.py b/test/functional/rpc_signrawtransactionwithkey.py index 268584331e..f4fec13495 100755 --- a/test/functional/rpc_signrawtransactionwithkey.py +++ b/test/functional/rpc_signrawtransactionwithkey.py @@ -126,10 +126,20 @@ class SignRawTransactionWithKeyTest(BitcoinTestFramework): privkeys = [self.nodes[0].get_deterministic_priv_key().key] assert_raises_rpc_error(-8, "'all' is not a valid sighash parameter.", self.nodes[0].signrawtransactionwithkey, tx, privkeys, sighashtype="all") + def invalid_private_key_and_tx(self): + self.log.info("Test signing transaction with an invalid private key") + tx = self.nodes[0].createrawtransaction(INPUTS, OUTPUTS) + privkeys = ["123"] + assert_raises_rpc_error(-5, "Invalid private key", self.nodes[0].signrawtransactionwithkey, tx, privkeys) + self.log.info("Test signing transaction with an invalid tx hex") + privkeys = [self.nodes[0].get_deterministic_priv_key().key] + assert_raises_rpc_error(-22, "TX decode failed. Make sure the tx has at least one input.", self.nodes[0].signrawtransactionwithkey, tx + "00", privkeys) + def run_test(self): self.successful_signing_test() self.witness_script_test() self.invalid_sighashtype_test() + self.invalid_private_key_and_tx() if __name__ == '__main__': diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index 7edf9f3679..a357ae4d34 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -26,7 +26,7 @@ ServiceProxy class: - HTTP connections persist for the life of the AuthServiceProxy object (if server supports HTTP/1.1) -- sends protocol 'version', per JSON-RPC 1.1 +- sends "jsonrpc":"2.0", per JSON-RPC 2.0 - sends proper, incrementing 'id' - sends Basic HTTP authentication headers - parses all JSON numbers that look like floats as Decimal @@ -117,7 +117,7 @@ class AuthServiceProxy(): params = dict(args=args, **argsn) else: params = args or argsn - return {'version': '1.1', + return {'jsonrpc': '2.0', 'method': self._service_name, 'params': params, 'id': AuthServiceProxy.__id_count} @@ -125,15 +125,28 @@ class AuthServiceProxy(): def __call__(self, *args, **argsn): postdata = json.dumps(self.get_request(*args, **argsn), default=serialization_fallback, ensure_ascii=self.ensure_ascii) response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) - if response['error'] is not None: - raise JSONRPCException(response['error'], status) - elif 'result' not in response: - raise JSONRPCException({ - 'code': -343, 'message': 'missing JSON-RPC result'}, status) - elif status != HTTPStatus.OK: - raise JSONRPCException({ - 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) + # For backwards compatibility tests, accept JSON RPC 1.1 responses + if 'jsonrpc' not in response: + if response['error'] is not None: + raise JSONRPCException(response['error'], status) + elif 'result' not in response: + raise JSONRPCException({ + 'code': -343, 'message': 'missing JSON-RPC result'}, status) + elif status != HTTPStatus.OK: + raise JSONRPCException({ + 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) + else: + return response['result'] else: + assert response['jsonrpc'] == '2.0' + if status != HTTPStatus.OK: + raise JSONRPCException({ + 'code': -342, 'message': 'non-200 HTTP status code'}, status) + if 'error' in response: + raise JSONRPCException(response['error'], status) + elif 'result' not in response: + raise JSONRPCException({ + 'code': -343, 'message': 'missing JSON-RPC 2.0 result and error'}, status) return response['result'] def batch(self, rpc_call_list): @@ -142,7 +155,7 @@ class AuthServiceProxy(): response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) if status != HTTPStatus.OK: raise JSONRPCException({ - 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) + 'code': -342, 'message': 'non-200 HTTP status code'}, status) return response def _get_response(self): diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 4e496a9275..005f7546a8 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -560,12 +560,12 @@ class CTxWitness: class CTransaction: - __slots__ = ("hash", "nLockTime", "nVersion", "sha256", "vin", "vout", + __slots__ = ("hash", "nLockTime", "version", "sha256", "vin", "vout", "wit") def __init__(self, tx=None): if tx is None: - self.nVersion = 2 + self.version = 2 self.vin = [] self.vout = [] self.wit = CTxWitness() @@ -573,7 +573,7 @@ class CTransaction: self.sha256 = None self.hash = None else: - self.nVersion = tx.nVersion + self.version = tx.version self.vin = copy.deepcopy(tx.vin) self.vout = copy.deepcopy(tx.vout) self.nLockTime = tx.nLockTime @@ -582,7 +582,7 @@ class CTransaction: self.wit = copy.deepcopy(tx.wit) def deserialize(self, f): - self.nVersion = int.from_bytes(f.read(4), "little", signed=True) + self.version = int.from_bytes(f.read(4), "little") self.vin = deser_vector(f, CTxIn) flags = 0 if len(self.vin) == 0: @@ -605,7 +605,7 @@ class CTransaction: def serialize_without_witness(self): r = b"" - r += self.nVersion.to_bytes(4, "little", signed=True) + r += self.version.to_bytes(4, "little") r += ser_vector(self.vin) r += ser_vector(self.vout) r += self.nLockTime.to_bytes(4, "little") @@ -617,7 +617,7 @@ class CTransaction: if not self.wit.is_null(): flags |= 1 r = b"" - r += self.nVersion.to_bytes(4, "little", signed=True) + r += self.version.to_bytes(4, "little") if flags: dummy = [] r += ser_vector(dummy) @@ -677,8 +677,8 @@ class CTransaction: return math.ceil(self.get_weight() / WITNESS_SCALE_FACTOR) def __repr__(self): - return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \ - % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime) + return "CTransaction(version=%i vin=%s vout=%s wit=%s nLockTime=%i)" \ + % (self.version, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime) class CBlockHeader: diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index 00bd1e4017..4b846df94a 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -411,7 +411,7 @@ class P2PConnection(asyncio.Protocol): tmsg = self.magic_bytes tmsg += msgtype tmsg += b"\x00" * (12 - len(msgtype)) - tmsg += struct.pack("<I", len(data)) + tmsg += len(data).to_bytes(4, "little") th = sha256(data) h = sha256(th) tmsg += h[:4] diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 7b19d31e17..97d62f957b 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -8,7 +8,6 @@ This file is modified from python-bitcoinlib. """ from collections import namedtuple -import struct import unittest from .key import TaggedHash, tweak_add_pubkey, compute_xonly_pubkey @@ -58,9 +57,9 @@ class CScriptOp(int): elif len(d) <= 0xff: return b'\x4c' + bytes([len(d)]) + d # OP_PUSHDATA1 elif len(d) <= 0xffff: - return b'\x4d' + struct.pack(b'<H', len(d)) + d # OP_PUSHDATA2 + return b'\x4d' + len(d).to_bytes(2, "little") + d # OP_PUSHDATA2 elif len(d) <= 0xffffffff: - return b'\x4e' + struct.pack(b'<I', len(d)) + d # OP_PUSHDATA4 + return b'\x4e' + len(d).to_bytes(4, "little") + d # OP_PUSHDATA4 else: raise ValueError("Data too long to encode in a PUSHDATA op") @@ -670,7 +669,7 @@ def LegacySignatureMsg(script, txTo, inIdx, hashtype): txtmp.vin.append(tmp) s = txtmp.serialize_without_witness() - s += struct.pack(b"<I", hashtype) + s += hashtype.to_bytes(4, "little") return (s, None) @@ -726,7 +725,7 @@ def SegwitV0SignatureMsg(script, txTo, inIdx, hashtype, amount): if (not (hashtype & SIGHASH_ANYONECANPAY) and (hashtype & 0x1f) != SIGHASH_SINGLE and (hashtype & 0x1f) != SIGHASH_NONE): serialize_sequence = bytes() for i in txTo.vin: - serialize_sequence += struct.pack("<I", i.nSequence) + serialize_sequence += i.nSequence.to_bytes(4, "little") hashSequence = uint256_from_str(hash256(serialize_sequence)) if ((hashtype & 0x1f) != SIGHASH_SINGLE and (hashtype & 0x1f) != SIGHASH_NONE): @@ -739,16 +738,16 @@ def SegwitV0SignatureMsg(script, txTo, inIdx, hashtype, amount): hashOutputs = uint256_from_str(hash256(serialize_outputs)) ss = bytes() - ss += struct.pack("<i", txTo.nVersion) + ss += txTo.version.to_bytes(4, "little") ss += ser_uint256(hashPrevouts) ss += ser_uint256(hashSequence) ss += txTo.vin[inIdx].prevout.serialize() ss += ser_string(script) - ss += struct.pack("<q", amount) - ss += struct.pack("<I", txTo.vin[inIdx].nSequence) + ss += amount.to_bytes(8, "little", signed=True) + ss += txTo.vin[inIdx].nSequence.to_bytes(4, "little") ss += ser_uint256(hashOutputs) ss += txTo.nLockTime.to_bytes(4, "little") - ss += struct.pack("<I", hashtype) + ss += hashtype.to_bytes(4, "little") return ss def SegwitV0SignatureHash(*args, **kwargs): @@ -800,13 +799,13 @@ def BIP341_sha_prevouts(txTo): return sha256(b"".join(i.prevout.serialize() for i in txTo.vin)) def BIP341_sha_amounts(spent_utxos): - return sha256(b"".join(struct.pack("<q", u.nValue) for u in spent_utxos)) + return sha256(b"".join(u.nValue.to_bytes(8, "little", signed=True) for u in spent_utxos)) def BIP341_sha_scriptpubkeys(spent_utxos): return sha256(b"".join(ser_string(u.scriptPubKey) for u in spent_utxos)) def BIP341_sha_sequences(txTo): - return sha256(b"".join(struct.pack("<I", i.nSequence) for i in txTo.vin)) + return sha256(b"".join(i.nSequence.to_bytes(4, "little") for i in txTo.vin)) def BIP341_sha_outputs(txTo): return sha256(b"".join(o.serialize() for o in txTo.vout)) @@ -818,8 +817,8 @@ def TaprootSignatureMsg(txTo, spent_utxos, hash_type, input_index = 0, scriptpat in_type = hash_type & SIGHASH_ANYONECANPAY spk = spent_utxos[input_index].scriptPubKey ss = bytes([0, hash_type]) # epoch, hash_type - ss += struct.pack("<i", txTo.nVersion) - ss += struct.pack("<I", txTo.nLockTime) + ss += txTo.version.to_bytes(4, "little") + ss += txTo.nLockTime.to_bytes(4, "little") if in_type != SIGHASH_ANYONECANPAY: ss += BIP341_sha_prevouts(txTo) ss += BIP341_sha_amounts(spent_utxos) @@ -835,11 +834,11 @@ def TaprootSignatureMsg(txTo, spent_utxos, hash_type, input_index = 0, scriptpat ss += bytes([spend_type]) if in_type == SIGHASH_ANYONECANPAY: ss += txTo.vin[input_index].prevout.serialize() - ss += struct.pack("<q", spent_utxos[input_index].nValue) + ss += spent_utxos[input_index].nValue.to_bytes(8, "little", signed=True) ss += ser_string(spk) - ss += struct.pack("<I", txTo.vin[input_index].nSequence) + ss += txTo.vin[input_index].nSequence.to_bytes(4, "little") else: - ss += struct.pack("<I", input_index) + ss += input_index.to_bytes(4, "little") if (spend_type & 1): ss += sha256(ser_string(annex)) if out_type == SIGHASH_SINGLE: @@ -850,7 +849,7 @@ def TaprootSignatureMsg(txTo, spent_utxos, hash_type, input_index = 0, scriptpat if (scriptpath): ss += TaggedHash("TapLeaf", bytes([leaf_ver]) + ser_string(script)) ss += bytes([0]) - ss += struct.pack("<i", codeseparator_pos) + ss += codeseparator_pos.to_bytes(4, "little", signed=True) assert len(ss) == 175 - (in_type == SIGHASH_ANYONECANPAY) * 49 - (out_type != SIGHASH_ALL and out_type != SIGHASH_SINGLE) * 32 + (annex is not None) * 32 + scriptpath * 37 return ss diff --git a/test/functional/test_framework/script_util.py b/test/functional/test_framework/script_util.py index 62894cc0f4..855f3b8cf5 100755 --- a/test/functional/test_framework/script_util.py +++ b/test/functional/test_framework/script_util.py @@ -3,10 +3,13 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Useful Script constants and utils.""" +import unittest + from test_framework.script import ( CScript, - CScriptOp, OP_0, + OP_15, + OP_16, OP_CHECKMULTISIG, OP_CHECKSIG, OP_DUP, @@ -49,10 +52,8 @@ def keys_to_multisig_script(keys, *, k=None): if k is None: # n-of-n multisig by default k = n assert k <= n - op_k = CScriptOp.encode_op_n(k) - op_n = CScriptOp.encode_op_n(n) checked_keys = [check_key(key) for key in keys] - return CScript([op_k] + checked_keys + [op_n, OP_CHECKMULTISIG]) + return CScript([k] + checked_keys + [n, OP_CHECKMULTISIG]) def keyhash_to_p2pkh_script(hash): @@ -125,3 +126,19 @@ def check_script(script): if isinstance(script, bytes) or isinstance(script, CScript): return script assert False + + +class TestFrameworkScriptUtil(unittest.TestCase): + def test_multisig(self): + fake_pubkey = bytes([0]*33) + # check correct encoding of P2MS script with n,k <= 16 + normal_ms_script = keys_to_multisig_script([fake_pubkey]*16, k=15) + self.assertEqual(len(normal_ms_script), 1 + 16*34 + 1 + 1) + self.assertTrue(normal_ms_script.startswith(bytes([OP_15]))) + self.assertTrue(normal_ms_script.endswith(bytes([OP_16, OP_CHECKMULTISIG]))) + + # check correct encoding of P2MS script with n,k > 16 + max_ms_script = keys_to_multisig_script([fake_pubkey]*20, k=19) + self.assertEqual(len(max_ms_script), 2 + 20*34 + 2 + 1) + self.assertTrue(max_ms_script.startswith(bytes([1, 19]))) # using OP_PUSH1 + self.assertTrue(max_ms_script.endswith(bytes([1, 20, OP_CHECKMULTISIG]))) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 620eeef270..9e44a11143 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2022 The Bitcoin Core developers +# Copyright (c) 2014-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Base class for RPC testing.""" @@ -444,6 +444,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): n.createwallet(wallet_name=wallet_name, descriptors=self.options.descriptors, load_on_startup=True) n.importprivkey(privkey=n.get_deterministic_priv_key().key, label='coinbase', rescan=True) + # Only enables wallet support when the module is available + def enable_wallet_if_possible(self): + self._requires_wallet = self.is_wallet_compiled() + def run_test(self): """Tests must override this method to define test logic""" raise NotImplementedError @@ -632,11 +636,6 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): def find_conn(node, peer_subversion, inbound): return next(filter(lambda peer: peer['subver'] == peer_subversion and peer['inbound'] == inbound, node.getpeerinfo()), None) - # poll until version handshake complete to avoid race conditions - # with transaction relaying - # See comments in net_processing: - # * Must have a version message before anything else - # * Must have a verack message before anything else self.wait_until(lambda: find_conn(from_connection, to_connection_subver, inbound=False) is not None) self.wait_until(lambda: find_conn(to_connection, from_connection_subver, inbound=True) is not None) @@ -644,11 +643,12 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): assert peer is not None, "Error: peer disconnected" return peer['bytesrecv_per_msg'].pop(msg_type, 0) >= min_bytes_recv - self.wait_until(lambda: check_bytesrecv(find_conn(from_connection, to_connection_subver, inbound=False), 'verack', 21)) - self.wait_until(lambda: check_bytesrecv(find_conn(to_connection, from_connection_subver, inbound=True), 'verack', 21)) - - # The message bytes are counted before processing the message, so make - # sure it was fully processed by waiting for a ping. + # Poll until version handshake (fSuccessfullyConnected) is complete to + # avoid race conditions, because some message types are blocked from + # being sent or received before fSuccessfullyConnected. + # + # As the flag fSuccessfullyConnected is not exposed, check it by + # waiting for a pong, which can only happen after the flag was set. self.wait_until(lambda: check_bytesrecv(find_conn(from_connection, to_connection_subver, inbound=False), 'pong', 29)) self.wait_until(lambda: check_bytesrecv(find_conn(to_connection, from_connection_subver, inbound=True), 'pong', 29)) diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 94ae96312e..0ac0af27d5 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -241,7 +241,7 @@ class TestNode(): if self.start_perf: self._start_perf() - def wait_for_rpc_connection(self): + def wait_for_rpc_connection(self, *, wait_for_import=True): """Sets up an RPC connection to the bitcoind process. Returns False if unable to connect.""" # Poll at a rate of four times per second poll_per_s = 4 @@ -263,7 +263,7 @@ class TestNode(): ) rpc.getblockcount() # If the call to getblockcount() succeeds then the RPC connection is up - if self.version_is_at_least(190000): + if self.version_is_at_least(190000) and wait_for_import: # getmempoolinfo.loaded is available since commit # bb8ae2c (version 0.19.0) self.wait_until(lambda: rpc.getmempoolinfo()['loaded']) diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index 4433cbcc55..cb0d291361 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -7,6 +7,7 @@ from copy import deepcopy from decimal import Decimal from enum import Enum +import math from typing import ( Any, Optional, @@ -33,10 +34,13 @@ from test_framework.messages import ( CTxInWitness, CTxOut, hash256, + ser_compact_size, + WITNESS_SCALE_FACTOR, ) from test_framework.script import ( CScript, LEAF_VERSION_TAPSCRIPT, + OP_1, OP_NOP, OP_RETURN, OP_TRUE, @@ -52,6 +56,7 @@ from test_framework.script_util import ( from test_framework.util import ( assert_equal, assert_greater_than_or_equal, + get_fee, ) from test_framework.wallet_util import generate_keypair @@ -119,13 +124,16 @@ class MiniWallet: """Pad a transaction with extra outputs until it reaches a target weight (or higher). returns the tx """ - tx.vout.append(CTxOut(nValue=0, scriptPubKey=CScript([OP_RETURN, b'a']))) + tx.vout.append(CTxOut(nValue=0, scriptPubKey=CScript([OP_RETURN]))) + # determine number of needed padding bytes by converting weight difference to vbytes dummy_vbytes = (target_weight - tx.get_weight() + 3) // 4 - tx.vout[-1].scriptPubKey = CScript([OP_RETURN, b'a' * dummy_vbytes]) - # Lower bound should always be off by at most 3 + # compensate for the increase of the compact-size encoded script length + # (note that the length encoding of the unpadded output script needs one byte) + dummy_vbytes -= len(ser_compact_size(dummy_vbytes)) - 1 + tx.vout[-1].scriptPubKey = CScript([OP_RETURN] + [OP_1] * dummy_vbytes) + # Actual weight should be at most 3 higher than target weight assert_greater_than_or_equal(tx.get_weight(), target_weight) - # Higher bound should always be off by at most 3 + 12 weight (for encoding the length) - assert_greater_than_or_equal(target_weight + 15, tx.get_weight()) + assert_greater_than_or_equal(target_weight + 3, tx.get_weight()) def get_balance(self): return sum(u['value'] for u in self._utxos) @@ -321,7 +329,7 @@ class MiniWallet: tx = CTransaction() tx.vin = [CTxIn(COutPoint(int(utxo_to_spend['txid'], 16), utxo_to_spend['vout']), nSequence=seq) for utxo_to_spend, seq in zip(utxos_to_spend, sequence)] tx.vout = [CTxOut(amount_per_output, bytearray(self._scriptPubKey)) for _ in range(num_outputs)] - tx.nVersion = version + tx.version = version tx.nLockTime = locktime self.sign_tx(tx) @@ -367,6 +375,10 @@ class MiniWallet: vsize = Decimal(168) # P2PK (73 bytes scriptSig + 35 bytes scriptPubKey + 60 bytes other) else: assert False + if target_weight and not fee: # respect fee_rate if target weight is passed + # the actual weight might be off by 3 WUs, so calculate based on that (see self._bulk_tx) + max_actual_weight = target_weight + 3 + fee = get_fee(math.ceil(max_actual_weight / WITNESS_SCALE_FACTOR), fee_rate) send_value = utxo_to_spend["value"] - (fee or (fee_rate * vsize / 1000)) # create tx diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 725b116281..8475dc5faa 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -15,8 +15,10 @@ For a description of arguments recognized by test scripts, see import argparse from collections import deque import configparser +import csv import datetime import os +import pathlib import platform import time import shutil @@ -280,6 +282,7 @@ BASE_SCRIPTS = [ 'mempool_packages.py', 'mempool_package_onemore.py', 'mempool_package_limits.py', + 'mempool_package_rbf.py', 'feature_versionbits_warning.py', 'rpc_preciousblock.py', 'wallet_importprunedfunds.py --legacy-wallet', @@ -361,6 +364,7 @@ BASE_SCRIPTS = [ 'feature_addrman.py', 'feature_asmap.py', 'feature_fastprune.py', + 'feature_framework_miniwallet.py', 'mempool_unbroadcast.py', 'mempool_compatibility.py', 'mempool_accept_wtxid.py', @@ -438,6 +442,7 @@ def main(): parser.add_argument('--filter', help='filter scripts to run by regular expression') parser.add_argument("--nocleanup", dest="nocleanup", default=False, action="store_true", help="Leave bitcoinds and test.* datadir on exit or error") + parser.add_argument('--resultsfile', '-r', help='store test results (as CSV) to the provided file') args, unknown_args = parser.parse_known_args() @@ -470,6 +475,13 @@ def main(): logging.debug("Temporary test directory at %s" % tmpdir) + results_filepath = None + if args.resultsfile: + results_filepath = pathlib.Path(args.resultsfile) + # Stop early if the parent directory doesn't exist + assert results_filepath.parent.exists(), "Results file parent directory does not exist" + logging.debug("Test results will be written to " + str(results_filepath)) + enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND") if not enable_bitcoind: @@ -556,9 +568,10 @@ def main(): combined_logs_len=args.combinedlogslen, failfast=args.failfast, use_term_control=args.ansi, + results_filepath=results_filepath, ) -def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage=False, args=None, combined_logs_len=0, failfast=False, use_term_control): +def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage=False, args=None, combined_logs_len=0, failfast=False, use_term_control, results_filepath=None): args = args or [] # Warn if bitcoind is already running @@ -650,11 +663,14 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= break if "[Errno 28] No space left on device" in stdout: - sys.exit(f"Early exiting after test failure due to insuffient free space in {tmpdir}\n" + sys.exit(f"Early exiting after test failure due to insufficient free space in {tmpdir}\n" f"Test execution data left in {tmpdir}.\n" f"Additional storage is needed to execute testing.") - print_results(test_results, max_len_name, (int(time.time() - start_time))) + runtime = int(time.time() - start_time) + print_results(test_results, max_len_name, runtime) + if results_filepath: + write_results(test_results, results_filepath, runtime) if coverage: coverage_passed = coverage.report_rpc_coverage() @@ -701,6 +717,17 @@ def print_results(test_results, max_len_name, runtime): results += "Runtime: %s s\n" % (runtime) print(results) + +def write_results(test_results, filepath, total_runtime): + with open(filepath, mode="w", encoding="utf8") as results_file: + results_writer = csv.writer(results_file) + results_writer.writerow(['test', 'status', 'duration(seconds)']) + all_passed = True + for test_result in test_results: + all_passed = all_passed and test_result.was_successful + results_writer.writerow([test_result.name, test_result.status, str(test_result.time)]) + results_writer.writerow(['ALL', ("Passed" if all_passed else "Failed"), str(total_runtime)]) + class TestHandler: """ Trigger the test scripts passed in via the list. diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index c322ae52c1..2c85773bf3 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet balance RPC methods.""" from decimal import Decimal -import struct from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE as ADDRESS_WATCHONLY from test_framework.blocktools import COINBASE_MATURITY @@ -266,8 +265,8 @@ class WalletTest(BitcoinTestFramework): tx_orig = self.nodes[0].gettransaction(txid)['hex'] # Increase fee by 1 coin tx_replace = tx_orig.replace( - struct.pack("<q", 99 * 10**8).hex(), - struct.pack("<q", 98 * 10**8).hex(), + (99 * 10**8).to_bytes(8, "little", signed=True).hex(), + (98 * 10**8).to_bytes(8, "little", signed=True).hex(), ) tx_replace = self.nodes[0].signrawtransactionwithwallet(tx_replace)['hex'] # Total balance is given by the sum of outputs of the tx diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 5b7db55f45..6d45adc823 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -117,6 +117,7 @@ class BumpFeeTest(BitcoinTestFramework): # Context independent tests test_feerate_checks_replaced_outputs(self, rbf_node, peer_node) + test_bumpfee_with_feerate_ignores_walletincrementalrelayfee(self, rbf_node, peer_node) def test_invalid_parameters(self, rbf_node, peer_node, dest_address): self.log.info('Test invalid parameters') @@ -816,7 +817,7 @@ def test_feerate_checks_replaced_outputs(self, rbf_node, peer_node): # Since the bumped tx will replace all of the outputs with a single output, we can estimate that its size will 31 * (len(outputs) - 1) bytes smaller tx_size = tx_details["decoded"]["vsize"] est_bumped_size = tx_size - (len(tx_details["decoded"]["vout"]) - 1) * 31 - inc_fee_rate = max(rbf_node.getmempoolinfo()["incrementalrelayfee"], Decimal(0.00005000)) # Wallet has a fixed incremental relay fee of 5 sat/vb + inc_fee_rate = rbf_node.getmempoolinfo()["incrementalrelayfee"] # RPC gives us fee as negative min_fee = (-tx_details["fee"] + get_fee(est_bumped_size, inc_fee_rate)) * Decimal(1e8) min_fee_rate = (min_fee / est_bumped_size).quantize(Decimal("1.000")) @@ -830,5 +831,27 @@ def test_feerate_checks_replaced_outputs(self, rbf_node, peer_node): self.clear_mempool() +def test_bumpfee_with_feerate_ignores_walletincrementalrelayfee(self, rbf_node, peer_node): + self.log.info('Test that bumpfee with fee_rate ignores walletincrementalrelayfee') + # Make sure there is enough balance + peer_node.sendtoaddress(rbf_node.getnewaddress(), 2) + self.generate(peer_node, 1) + + dest_address = peer_node.getnewaddress(address_type="bech32") + tx = rbf_node.send(outputs=[{dest_address: 1}], fee_rate=2) + + # Ensure you can not fee bump with a fee_rate below or equal to the original fee_rate + assert_raises_rpc_error(-8, "Insufficient total fee", rbf_node.bumpfee, tx["txid"], {"fee_rate": 1}) + assert_raises_rpc_error(-8, "Insufficient total fee", rbf_node.bumpfee, tx["txid"], {"fee_rate": 2}) + + # Ensure you can not fee bump if the fee_rate is more than original fee_rate but the total fee from new fee_rate is + # less than (original fee + incrementalrelayfee) + assert_raises_rpc_error(-8, "Insufficient total fee", rbf_node.bumpfee, tx["txid"], {"fee_rate": 2.8}) + + # You can fee bump as long as the new fee set from fee_rate is atleast (original fee + incrementalrelayfee) + rbf_node.bumpfee(tx["txid"], {"fee_rate": 3}) + self.clear_mempool() + + if __name__ == "__main__": BumpFeeTest().main() diff --git a/test/functional/wallet_create_tx.py b/test/functional/wallet_create_tx.py index 4e31b48ec0..fa3e920c25 100755 --- a/test/functional/wallet_create_tx.py +++ b/test/functional/wallet_create_tx.py @@ -3,6 +3,9 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +from test_framework.messages import ( + tx_from_hex, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -33,6 +36,7 @@ class CreateTxWalletTest(BitcoinTestFramework): self.test_anti_fee_sniping() self.test_tx_size_too_large() self.test_create_too_long_mempool_chain() + self.test_version3() def test_anti_fee_sniping(self): self.log.info('Check that we have some (old) blocks and that anti-fee-sniping is disabled') @@ -106,6 +110,23 @@ class CreateTxWalletTest(BitcoinTestFramework): test_wallet.unloadwallet() + def test_version3(self): + self.log.info('Check wallet does not create transactions with version=3 yet') + wallet_rpc = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + + self.nodes[0].createwallet("v3") + wallet_v3 = self.nodes[0].get_wallet_rpc("v3") + + tx_data = wallet_rpc.send(outputs=[{wallet_v3.getnewaddress(): 25}], options={"change_position": 0}) + wallet_tx_data = wallet_rpc.gettransaction(tx_data["txid"]) + tx_current_version = tx_from_hex(wallet_tx_data["hex"]) + + # While v3 transactions are standard, the CURRENT_VERSION is 2. + # This test can be removed if CURRENT_VERSION is changed, and replaced with tests that the + # wallet handles v3 rules properly. + assert_equal(tx_current_version.version, 2) + wallet_v3.unloadwallet() + if __name__ == '__main__': CreateTxWalletTest().main() diff --git a/test/functional/wallet_listsinceblock.py b/test/functional/wallet_listsinceblock.py index fd586d546e..15214539a9 100755 --- a/test/functional/wallet_listsinceblock.py +++ b/test/functional/wallet_listsinceblock.py @@ -40,6 +40,7 @@ class ListSinceBlockTest(BitcoinTestFramework): self.test_no_blockhash() self.test_invalid_blockhash() self.test_reorg() + self.test_cant_read_block() self.test_double_spend() self.test_double_send() self.double_spends_filtered() @@ -167,6 +168,31 @@ class ListSinceBlockTest(BitcoinTestFramework): found = next(tx for tx in transactions if tx['txid'] == senttx) assert_equal(found['blockheight'], self.nodes[0].getblockheader(nodes2_first_blockhash)['height']) + def test_cant_read_block(self): + self.log.info('Test the RPC error "Can\'t read block from disk"') + + # Split network into two + self.split_network() + + # generate on both sides + nodes1_last_blockhash = self.generate(self.nodes[1], 6, sync_fun=lambda: self.sync_all(self.nodes[:2]))[-1] + self.generate(self.nodes[2], 7, sync_fun=lambda: self.sync_all(self.nodes[2:]))[0] + + self.join_network() + + # Renaming the block file to induce unsuccessful block read + blk_dat = (self.nodes[0].blocks_path / "blk00000.dat") + blk_dat_moved = blk_dat.rename(self.nodes[0].blocks_path / "blk00000.dat.moved") + assert not blk_dat.exists() + + # listsinceblock(nodes1_last_blockhash) should now fail as blocks are not accessible + assert_raises_rpc_error(-32603, "Can't read block from disk", + self.nodes[0].listsinceblock, nodes1_last_blockhash) + + # Restoring block file + blk_dat_moved.rename(self.nodes[0].blocks_path / "blk00000.dat") + assert blk_dat.exists() + def test_double_spend(self): ''' This tests the case where the same UTXO is spent twice on two separate diff --git a/test/functional/wallet_sendall.py b/test/functional/wallet_sendall.py index c2b800df21..1d308c225d 100755 --- a/test/functional/wallet_sendall.py +++ b/test/functional/wallet_sendall.py @@ -379,6 +379,64 @@ class SendallTest(BitcoinTestFramework): assert_equal(len(self.wallet.listunspent()), 1) assert_equal(self.wallet.listunspent()[0]['confirmations'], 6) + @cleanup + def sendall_spends_unconfirmed_change(self): + self.log.info("Test that sendall spends unconfirmed change") + self.add_utxos([17]) + self.wallet.sendtoaddress(self.remainder_target, 10) + assert_greater_than(self.wallet.getbalances()["mine"]["trusted"], 6) + self.test_sendall_success(sendall_args = [self.remainder_target]) + + assert_equal(self.wallet.getbalance(), 0) + + @cleanup + def sendall_spends_unconfirmed_inputs_if_specified(self): + self.log.info("Test that sendall spends specified unconfirmed inputs") + self.def_wallet.sendtoaddress(self.wallet.getnewaddress(), 17) + self.wallet.syncwithvalidationinterfacequeue() + assert_equal(self.wallet.getbalances()["mine"]["untrusted_pending"], 17) + unspent = self.wallet.listunspent(minconf=0)[0] + + self.wallet.sendall(recipients=[self.remainder_target], inputs=[unspent]) + assert_equal(self.wallet.getbalance(), 0) + + @cleanup + def sendall_does_ancestor_aware_funding(self): + self.log.info("Test that sendall does ancestor aware funding for unconfirmed inputs") + + # higher parent feerate + self.def_wallet.sendtoaddress(address=self.wallet.getnewaddress(), amount=17, fee_rate=20) + self.wallet.syncwithvalidationinterfacequeue() + + assert_equal(self.wallet.getbalances()["mine"]["untrusted_pending"], 17) + unspent = self.wallet.listunspent(minconf=0)[0] + + parent_txid = unspent["txid"] + assert_equal(self.wallet.gettransaction(parent_txid)["confirmations"], 0) + + res_1 = self.wallet.sendall(recipients=[self.def_wallet.getnewaddress()], inputs=[unspent], fee_rate=20, add_to_wallet=False, lock_unspents=True) + child_hex = res_1["hex"] + + child_tx = self.wallet.decoderawtransaction(child_hex) + higher_parent_feerate_amount = child_tx["vout"][0]["value"] + + # lower parent feerate + self.def_wallet.sendtoaddress(address=self.wallet.getnewaddress(), amount=17, fee_rate=10) + self.wallet.syncwithvalidationinterfacequeue() + assert_equal(self.wallet.getbalances()["mine"]["untrusted_pending"], 34) + unspent = self.wallet.listunspent(minconf=0)[0] + + parent_txid = unspent["txid"] + assert_equal(self.wallet.gettransaction(parent_txid)["confirmations"], 0) + + res_2 = self.wallet.sendall(recipients=[self.def_wallet.getnewaddress()], inputs=[unspent], fee_rate=20, add_to_wallet=False, lock_unspents=True) + child_hex = res_2["hex"] + + child_tx = self.wallet.decoderawtransaction(child_hex) + lower_parent_feerate_amount = child_tx["vout"][0]["value"] + + assert_greater_than(higher_parent_feerate_amount, lower_parent_feerate_amount) + # This tests needs to be the last one otherwise @cleanup will fail with "Transaction too large" error def sendall_fails_with_transaction_too_large(self): self.log.info("Test that sendall fails if resulting transaction is too large") @@ -460,6 +518,15 @@ class SendallTest(BitcoinTestFramework): # Sendall only uses outputs with less than a given number of confirmation when using minconf self.sendall_with_maxconf() + # Sendall spends unconfirmed change + self.sendall_spends_unconfirmed_change() + + # Sendall spends unconfirmed inputs if they are specified + self.sendall_spends_unconfirmed_inputs_if_specified() + + # Sendall does ancestor aware funding when spending an unconfirmed UTXO + self.sendall_does_ancestor_aware_funding() + # Sendall fails when many inputs result to too large transaction self.sendall_fails_with_transaction_too_large() diff --git a/test/lint/README.md b/test/lint/README.md index 49ed8356c3..04a836c4d2 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -26,6 +26,21 @@ Then you can use: ( cd ./test/lint/test_runner/ && cargo fmt && cargo clippy && RUST_BACKTRACE=1 cargo run ) ``` +If you wish to run individual lint checks, run the test_runner with +`--lint=TEST_TO_RUN` arguments. If running with `cargo run`, arguments after +`--` are passed to the binary you are running e.g.: + +```sh +( cd ./test/lint/test_runner/ && RUST_BACKTRACE=1 cargo run -- --lint=doc --lint=trailing_whitespace ) +``` + +To see a list of all individual lint checks available in test_runner, use `-h` +or `--help`: + +```sh +( cd ./test/lint/test_runner/ && RUST_BACKTRACE=1 cargo run -- --help ) +``` + #### Dependencies | Lint test | Dependency | diff --git a/test/lint/test_runner/src/main.rs b/test/lint/test_runner/src/main.rs index 5f980eb398..9c35898c1f 100644 --- a/test/lint/test_runner/src/main.rs +++ b/test/lint/test_runner/src/main.rs @@ -12,6 +12,113 @@ type LintError = String; type LintResult = Result<(), LintError>; type LintFn = fn() -> LintResult; +struct Linter { + pub description: &'static str, + pub name: &'static str, + pub lint_fn: LintFn, +} + +fn get_linter_list() -> Vec<&'static Linter> { + vec![ + &Linter { + description: "Check that all command line arguments are documented.", + name: "doc", + lint_fn: lint_doc + }, + &Linter { + description: "Check that no symbol from bitcoin-config.h is used without the header being included", + name: "includes_build_config", + lint_fn: lint_includes_build_config + }, + &Linter { + description: "Check that markdown links resolve", + name: "markdown", + lint_fn: lint_markdown + }, + &Linter { + description: "Check that std::filesystem is not used directly", + name: "std_filesystem", + lint_fn: lint_std_filesystem + }, + &Linter { + description: "Check that subtrees are pure subtrees", + name: "subtree", + lint_fn: lint_subtree + }, + &Linter { + description: "Check that tabs are not used as whitespace", + name: "tabs_whitespace", + lint_fn: lint_tabs_whitespace + }, + &Linter { + description: "Check for trailing whitespace", + name: "trailing_whitespace", + lint_fn: lint_trailing_whitespace + }, + &Linter { + description: "Run all linters of the form: test/lint/lint-*.py", + name: "all_python_linters", + lint_fn: run_all_python_linters + }, + ] +} + +fn print_help_and_exit() { + print!( + r#" +Usage: test_runner [--lint=LINTER_TO_RUN] +Runs all linters in the lint test suite, printing any errors +they detect. + +If you wish to only run some particular lint tests, pass +'--lint=' with the name of the lint test you wish to run. +You can set as many '--lint=' values as you wish, e.g.: +test_runner --lint=doc --lint=subtree + +The individual linters available to run are: +"# + ); + for linter in get_linter_list() { + println!("{}: \"{}\"", linter.name, linter.description) + } + + std::process::exit(1); +} + +fn parse_lint_args(args: &[String]) -> Vec<&'static Linter> { + let linter_list = get_linter_list(); + let mut lint_values = Vec::new(); + + for arg in args { + #[allow(clippy::if_same_then_else)] + if arg.starts_with("--lint=") { + let lint_arg_value = arg + .trim_start_matches("--lint=") + .trim_matches('"') + .trim_matches('\''); + + let try_find_linter = linter_list + .iter() + .find(|linter| linter.name == lint_arg_value); + match try_find_linter { + Some(linter) => { + lint_values.push(*linter); + } + None => { + println!("No linter {lint_arg_value} found!"); + print_help_and_exit(); + } + } + } else if arg.eq("--help") || arg.eq("-h") { + print_help_and_exit(); + } else { + print_help_and_exit(); + } + } + + lint_values +} + /// Return the git command fn git() -> Command { let mut git = Command::new("git"); @@ -337,7 +444,7 @@ Markdown link errors found: } } -fn lint_all() -> LintResult { +fn run_all_python_linters() -> LintResult { let mut good = true; let lint_dir = get_git_root().join("test/lint"); for entry in fs::read_dir(lint_dir).unwrap() { @@ -352,7 +459,7 @@ fn lint_all() -> LintResult { .success() { good = false; - println!("^---- failure generated from {}", entry_fn); + println!("^---- ⚠️ Failure generated from {}", entry_fn); } } if good { @@ -363,25 +470,26 @@ fn lint_all() -> LintResult { } fn main() -> ExitCode { - let test_list: Vec<(&str, LintFn)> = vec![ - ("subtree check", lint_subtree), - ("std::filesystem check", lint_std_filesystem), - ("trailing whitespace check", lint_trailing_whitespace), - ("no-tabs check", lint_tabs_whitespace), - ("build config includes check", lint_includes_build_config), - ("-help=1 documentation check", lint_doc), - ("markdown hyperlink check", lint_markdown), - ("lint-*.py scripts", lint_all), - ]; + let linters_to_run: Vec<&Linter> = if env::args().count() > 1 { + let args: Vec<String> = env::args().skip(1).collect(); + parse_lint_args(&args) + } else { + // If no arguments are passed, run all linters. + get_linter_list() + }; let git_root = get_git_root(); let mut test_failed = false; - for (lint_name, lint_fn) in test_list { + for linter in linters_to_run { // chdir to root before each lint test env::set_current_dir(&git_root).unwrap(); - if let Err(err) = lint_fn() { - println!("{err}\n^---- ⚠️ Failure generated from {lint_name}!"); + if let Err(err) = (linter.lint_fn)() { + println!( + "{err}\n^---- ⚠️ Failure generated from lint check '{}'!", + linter.name + ); + println!("{}", linter.description); test_failed = true; } } diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 482667a26a..9818d73fdf 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -58,6 +58,7 @@ unsigned-integer-overflow:TxConfirmStats::EstimateMedianVal unsigned-integer-overflow:prevector.h unsigned-integer-overflow:EvalScript unsigned-integer-overflow:xoroshiro128plusplus.h +unsigned-integer-overflow:bitset_detail::PopCount implicit-integer-sign-change:CBlockPolicyEstimator::processBlockTx implicit-integer-sign-change:SetStdinEcho implicit-integer-sign-change:compressor.h |