diff options
395 files changed, 7305 insertions, 3024 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 3c59e41a13..b37fce7002 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -8,27 +8,44 @@ env: # Global defaults CCACHE_DIR: "/tmp/ccache_dir" CCACHE_NOHASHDIR: "1" # Debug info might contain a stale path if the build dir changes, but this is fine +# A self-hosted machine(s) can be used via Cirrus CI. It can be configured with +# multiple users to run tasks in parallel. No sudo permission is required. +# # https://cirrus-ci.org/guide/persistent-workers/ # -# It is possible to select a specific persistent worker by label. Refer to the +# Generally, a persistent worker must run Ubuntu 23.04+ or Debian 12+. +# +# The following specific types should exist, with the following requirements: +# - small: For an x86_64 machine, recommended to have 2 CPUs and 8 GB of memory. +# - medium: For an x86_64 machine, recommended to have 4 CPUs and 16 GB of memory. +# - arm64: For an aarch64 machine, recommended to have 2 CPUs and 8 GB of memory. +# +# CI jobs for the latter configuration can be run on x86_64 hardware +# by installing qemu-user-static, which works out of the box with +# podman or docker. Background: https://stackoverflow.com/a/72890225/313633 +# +# The above machine types are matched to each task by their label. Refer to the # Cirrus CI docs for more details. # -# Generally, a persistent worker must run Ubuntu 23.04+ or Debian 12+. -# Specifically, +# On machines that are persisted between CI jobs, RESTART_CI_DOCKER_BEFORE_RUN=1 +# ensures that previous containers and artifacts are cleared before each run. +# This requires installing Podman instead of Docker. +# +# Futhermore: # - apt-get is required due to PACKAGE_MANAGER_INSTALL -# - podman-docker-4.1+ is required due to the use of `podman` when -# RESTART_CI_DOCKER_BEFORE_RUN is set and 4.1+ due to the bugfix in 4.1 +# - podman-docker-4.1+ is required due to the bugfix in 4.1 # (https://github.com/bitcoin/bitcoin/pull/21652#issuecomment-1657098200) -# - The ./ci/ depedencies (with cirrus-cli) should be installed: +# - The ./ci/ dependencies (with cirrus-cli) should be installed. One-liner example +# for a single user setup with sudo permission: # # ``` # apt update && apt install git screen python3 bash podman-docker curl -y && curl -L -o cirrus "https://github.com/cirruslabs/cirrus-cli/releases/latest/download/cirrus-linux-$(dpkg --print-architecture)" && mv cirrus /usr/local/bin/cirrus && chmod +x /usr/local/bin/cirrus # ``` # -# - There are no strict requirements on the hardware, because having less CPUs -# runs the same CI script (maybe slower). To avoid rare and intermittent OOM -# due to short memory usage spikes, it is recommended to add (and persist) -# swap: +# - There are no strict requirements on the hardware. Having fewer CPU threads +# than recommended merely causes the CI script to run slower. +# To avoid rare and intermittent OOM due to short memory usage spikes, +# it is recommended to add (and persist) swap: # # ``` # fallocate -l 16G /swapfile_ci && chmod 600 /swapfile_ci && mkswap /swapfile_ci && swapon /swapfile_ci && ( echo '/swapfile_ci none swap sw 0 0' | tee -a /etc/fstab ) @@ -39,12 +56,6 @@ env: # Global defaults # ``` # RESTART_CI_DOCKER_BEFORE_RUN=1 screen cirrus worker run --labels type=todo_fill_in_type --token todo_fill_in_token # ``` -# -# The following specific types should exist, with the following requirements: -# - small: For an x86_64 machine, recommended to have 2 CPUs and 8 GB of memory. -# - medium: For an x86_64 machine, recommended to have 4 CPUs and 16 GB of memory. -# - noble: For a machine running the Linux kernel shipped with exaclty Ubuntu Noble 24.04. The machine is recommended to have 4 CPUs and 16 GB of memory. -# - arm64: For an aarch64 machine, recommended to have 2 CPUs and 8 GB of memory. # https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks filter_template: &FILTER_TEMPLATE @@ -160,19 +171,6 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" task: - name: 'ASan + LSan + UBSan + integer, no depends, USDT' - enable_bpfcc_script: - # In the image build step, no external environment variables are available, - # so any settings will need to be written to the settings env file: - - sed -i "s|\${CIRRUS_CI}|true|g" ./ci/test/00_setup_env_native_asan.sh - << : *GLOBAL_TASK_TEMPLATE - persistent_worker: - labels: - type: noble # Must use this specific worker (needed for USDT functional tests) - env: - FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" - -task: name: 'fuzzer,address,undefined,integer, no depends' << : *GLOBAL_TASK_TEMPLATE persistent_worker: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5c322fc23..54795332e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,6 @@ concurrency: cancel-in-progress: true env: - DANGER_RUN_CI_ON_HOST: 1 CI_FAILFAST_TEST_LEAVE_DANGLING: 1 # GHA does not care about dangling processes and setting this variable avoids killing the CI script itself on error MAKEJOBS: '-j10' @@ -81,6 +80,7 @@ jobs: timeout-minutes: 120 env: + DANGER_RUN_CI_ON_HOST: 1 FILE_ENV: './ci/test/00_setup_env_mac_native.sh' BASE_ROOT_DIR: ${{ github.workspace }} @@ -153,12 +153,10 @@ jobs: - name: Get tool information run: | - msbuild -version | Out-File -FilePath "$env:GITHUB_WORKSPACE\msbuild_version" - Get-Content -Path "$env:GITHUB_WORKSPACE\msbuild_version" - $env:VCToolsVersion | Out-File -FilePath "$env:GITHUB_WORKSPACE\toolset_version" - Write-Host "VCToolsVersion $(Get-Content -Path "$env:GITHUB_WORKSPACE\toolset_version")" - $env:CI_QT_URL | Out-File -FilePath "$env:GITHUB_WORKSPACE\qt_url" - $env:CI_QT_CONF | Out-File -FilePath "$env:GITHUB_WORKSPACE\qt_conf" + msbuild -version | Tee-Object -FilePath "msbuild_version" + $env:VCToolsVersion | Tee-Object -FilePath "toolset_version" + $env:CI_QT_URL | Out-File -FilePath "qt_url" + $env:CI_QT_CONF | Out-File -FilePath "qt_conf" py -3 --version Write-Host "PowerShell version $($PSVersionTable.PSVersion.ToString())" @@ -241,10 +239,8 @@ jobs: run: | Set-Location "$env:VCPKG_INSTALLATION_ROOT" Add-Content -Path "triplets\x64-windows-static.cmake" -Value "set(VCPKG_BUILD_TYPE release)" - Add-Content -Path "triplets\x64-windows-static.cmake" -Value "set(VCPKG_PLATFORM_TOOLSET_VERSION $env:VCToolsVersion)" .\vcpkg.exe --vcpkg-root "$env:VCPKG_INSTALLATION_ROOT" integrate install - git rev-parse HEAD | Out-File -FilePath "$env:GITHUB_WORKSPACE\vcpkg_commit" - Get-Content -Path "$env:GITHUB_WORKSPACE\vcpkg_commit" + git rev-parse HEAD | Tee-Object -FilePath "$env:GITHUB_WORKSPACE\vcpkg_commit" - name: vcpkg tools cache uses: actions/cache@v4 @@ -308,3 +304,46 @@ jobs: BITCOINFUZZ: "${{ github.workspace}}\\src\\fuzz.exe" shell: cmd run: py -3 test\fuzz\test_runner.py --par %NUMBER_OF_PROCESSORS% --loglevel DEBUG %RUNNER_TEMP%\qa-assets\fuzz_seed_corpus + + asan-lsan-ubsan-integer-no-depends-usdt: + name: 'ASan + LSan + UBSan + integer, no depends, USDT' + runs-on: ubuntu-24.04 # has to match container in ci/test/00_setup_env_native_asan.sh for tracing tools + # No need to run on the read-only mirror, unless it is a PR. + if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request' + timeout-minutes: 120 + env: + FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" + DANGER_CI_ON_HOST_CACHE_FOLDERS: 1 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set Ccache directory + run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV" + + - name: Set base root directory + run: echo "BASE_ROOT_DIR=${RUNNER_TEMP}" >> "$GITHUB_ENV" + + - name: Restore Ccache cache + id: ccache-cache + uses: actions/cache/restore@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ${{ github.job }}-ccache-${{ github.run_id }} + restore-keys: ${{ github.job }}-ccache- + + - name: Enable bpfcc script + # In the image build step, no external environment variables are available, + # so any settings will need to be written to the settings env file: + run: sed -i "s|\${INSTALL_BCC_TRACING_TOOLS}|true|g" ./ci/test/00_setup_env_native_asan.sh + + - name: CI script + run: ./ci/test_run_all.sh + + - name: Save Ccache cache + uses: actions/cache/save@v4 + if: github.event_name != 'pull_request' && steps.ccache-cache.outputs.cache-hit != 'true' + with: + path: ${{ env.CCACHE_DIR }} + # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache + key: ${{ github.job }}-ccache-${{ github.run_id }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 76a1c2754a..f594172333 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -149,7 +149,7 @@ the pull request affects. Valid areas as: - `net` or `p2p` for changes to the peer-to-peer network code - `refactor` for structural changes that do not change behavior - `rpc`, `rest` or `zmq` for changes to the RPC, REST or ZMQ APIs - - `script` for changes to the scripts and tools + - `contrib` or `cli` for changes to the scripts and tools - `test`, `qa` or `ci` for changes to the unit tests, QA tests or CI code - `util` or `lib` for changes to the utils or libraries - `wallet` for changes to the wallet code diff --git a/build-aux/m4/ax_cxx_compile_stdcxx.m4 b/build-aux/m4/ax_cxx_compile_stdcxx.m4 index 8a2df5627f..51a35054d0 100644 --- a/build-aux/m4/ax_cxx_compile_stdcxx.m4 +++ b/build-aux/m4/ax_cxx_compile_stdcxx.m4 @@ -983,7 +983,7 @@ m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_20], [[ #error "This is not a C++ compiler" -#elif __cplusplus < 201709L // Temporary patch on top of upstream to allow g++-10 +#elif __cplusplus < 202002L #error "This is not a C++20 compiler" diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index 337e2208a9..b87c38a6ab 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -70,8 +70,6 @@ AC_DEFUN([BITCOIN_QT_INIT],[ [qt_lib_suffix= ]); bitcoin_qt_want_version=qt5], [qt_lib_suffix= ]) - AS_CASE([$host], [*android*], [qt_lib_suffix=_$ANDROID_ARCH]) - AC_ARG_WITH([qt-incdir],[AS_HELP_STRING([--with-qt-incdir=INC_DIR],[specify qt include path (overridden by pkgconfig)])], [qt_include_path=$withval], []) AC_ARG_WITH([qt-libdir],[AS_HELP_STRING([--with-qt-libdir=LIB_DIR],[specify qt lib path (overridden by pkgconfig)])], [qt_lib_path=$withval], []) AC_ARG_WITH([qt-plugindir],[AS_HELP_STRING([--with-qt-plugindir=PLUGIN_DIR],[specify qt plugin path (overridden by pkgconfig)])], [qt_plugin_path=$withval], []) @@ -80,19 +78,10 @@ AC_DEFUN([BITCOIN_QT_INIT],[ AC_ARG_WITH([qtdbus], [AS_HELP_STRING([--with-qtdbus], - [enable DBus support (default is yes if qt is enabled and QtDBus is found, except on Android)])], + [enable DBus support (default is yes if qt is enabled and QtDBus is found)])], [use_dbus=$withval], [use_dbus=auto]) - dnl Android doesn't support D-Bus and certainly doesn't use it for notifications - case $host in - *android*) - if test "$use_dbus" != "yes"; then - use_dbus=no - fi - ;; - esac - AC_SUBST(QT_TRANSLATION_DIR,$qt_translation_path) ]) @@ -132,16 +121,11 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ if test -d "$qt_plugin_path/accessible"; then QT_LIBS="$QT_LIBS -L$qt_plugin_path/accessible" fi - if test -d "$qt_plugin_path/platforms/android"; then - QT_LIBS="$QT_LIBS -L$qt_plugin_path/platforms/android -lqtfreetype -lEGL" - fi fi AC_DEFINE([QT_STATICPLUGIN], [1], [Define this symbol if qt plugins are static]) - if test "$TARGET_OS" != "android"; then - _BITCOIN_QT_CHECK_STATIC_PLUGIN([QMinimalIntegrationPlugin], [-lqminimal]) - AC_DEFINE([QT_QPA_PLATFORM_MINIMAL], [1], [Define this symbol if the minimal qt platform exists]) - fi + _BITCOIN_QT_CHECK_STATIC_PLUGIN([QMinimalIntegrationPlugin], [-lqminimal]) + AC_DEFINE([QT_QPA_PLATFORM_MINIMAL], [1], [Define this symbol if the minimal qt platform exists]) if test "$TARGET_OS" = "windows"; then dnl Linking against wtsapi32 is required. See #17749 and dnl https://bugreports.qt.io/browse/QTBUG-27097. @@ -160,9 +144,6 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ _BITCOIN_QT_CHECK_STATIC_PLUGIN([QCocoaIntegrationPlugin], [-lqcocoa]) _BITCOIN_QT_CHECK_STATIC_PLUGIN([QMacStylePlugin], [-lqmacstyle]) AC_DEFINE([QT_QPA_PLATFORM_COCOA], [1], [Define this symbol if the qt platform is cocoa]) - elif test "$TARGET_OS" = "android"; then - QT_LIBS="-Wl,--export-dynamic,--undefined=JNI_OnLoad -lplugins_platforms_qtforandroid${qt_lib_suffix} -ljnigraphics -landroid -lqtfreetype${qt_lib_suffix} $QT_LIBS" - AC_DEFINE([QT_QPA_PLATFORM_ANDROID], [1], [Define this symbol if the qt platform is android]) fi fi CPPFLAGS=$TEMP_CPPFLAGS @@ -357,9 +338,6 @@ AC_DEFUN([_BITCOIN_QT_CHECK_STATIC_LIBS], [ PKG_CHECK_MODULES([QT_SERVICE], [${qt_lib_prefix}ServiceSupport${qt_lib_suffix}], [QT_LIBS="$QT_SERVICE_LIBS $QT_LIBS"]) elif test "$TARGET_OS" = "windows"; then PKG_CHECK_MODULES([QT_WINDOWSUIAUTOMATION], [${qt_lib_prefix}WindowsUIAutomationSupport${qt_lib_suffix}], [QT_LIBS="$QT_WINDOWSUIAUTOMATION_LIBS $QT_LIBS"]) - elif test "$TARGET_OS" = "android"; then - PKG_CHECK_MODULES([QT_EGL], [${qt_lib_prefix}EglSupport${qt_lib_suffix}], [QT_LIBS="$QT_EGL_LIBS $QT_LIBS"]) - PKG_CHECK_MODULES([QT_SERVICE], [${qt_lib_prefix}ServiceSupport${qt_lib_suffix}], [QT_LIBS="$QT_SERVICE_LIBS $QT_LIBS"]) fi ]) diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index 6b12c53f2a..87e3a8fa9b 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -57,3 +57,8 @@ SHELLCHECK_VERSION=v0.8.0 curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | \ tar --xz -xf - --directory /tmp/ mv "/tmp/shellcheck-${SHELLCHECK_VERSION}/shellcheck" /usr/bin/ + +MLC_VERSION=v0.16.3 +MLC_BIN=mlc-x86_64-linux +curl -sL "https://github.com/becheran/mlc/releases/download/${MLC_VERSION}/${MLC_BIN}" -o "/usr/bin/mlc" +chmod +x /usr/bin/mlc diff --git a/ci/test/00_setup_env_android.sh b/ci/test/00_setup_env_android.sh deleted file mode 100755 index 97a6bd7318..0000000000 --- a/ci/test/00_setup_env_android.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019-present The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -export HOST=aarch64-linux-android -export PACKAGES="unzip openjdk-8-jdk gradle" -export CONTAINER_NAME=ci_android -export CI_IMAGE_NAME_TAG="docker.io/amd64/ubuntu:22.04" - -export RUN_UNIT_TESTS=false -export RUN_FUNCTIONAL_TESTS=false - -export ANDROID_API_LEVEL=28 -export ANDROID_BUILD_TOOLS_VERSION=28.0.3 -export ANDROID_NDK_VERSION=23.2.8568313 -export ANDROID_TOOLS_URL=https://dl.google.com/android/repository/commandlinetools-linux-8512546_latest.zip -export ANDROID_HOME="${DEPENDS_DIR}/SDKs/android" -export ANDROID_NDK_HOME="${ANDROID_HOME}/ndk/${ANDROID_NDK_VERSION}" -export DEP_OPTS="ANDROID_SDK=${ANDROID_HOME} ANDROID_NDK=${ANDROID_NDK_HOME} ANDROID_API_LEVEL=${ANDROID_API_LEVEL} ANDROID_TOOLCHAIN_BIN=${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/linux-x86_64/bin/" - -export BITCOIN_CONFIG="--disable-tests --enable-gui-tests --disable-bench --disable-fuzz-binary --without-utils --without-libs --without-daemon" diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index c80036164f..396e782959 100755 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -17,4 +17,4 @@ export RUN_FUNCTIONAL_TESTS=false export GOAL="install" # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1" # This could be removed once the ABI change warning does not show up by default -export BITCOIN_CONFIG="--enable-reduce-exports CXXFLAGS=-Wno-psabi" +export BITCOIN_CONFIG="--enable-reduce-exports CXXFLAGS='-Wno-psabi -Wno-error=maybe-uninitialized'" diff --git a/ci/test/00_setup_env_mac_cross.sh b/ci/test/00_setup_env_mac_cross.sh index 31c4bff6ae..f607c93ae6 100755 --- a/ci/test/00_setup_env_mac_cross.sh +++ b/ci/test/00_setup_env_mac_cross.sh @@ -9,9 +9,9 @@ export LC_ALL=C.UTF-8 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} export CONTAINER_NAME=ci_macos_cross -export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04" +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export HOST=x86_64-apple-darwin -export PACKAGES="zip" +export PACKAGES="clang lld llvm zip" export XCODE_VERSION=15.0 export XCODE_BUILD_ID=15A240d export RUN_UNIT_TESTS=false diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 668e9ecc8a..23d9180f96 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -7,8 +7,10 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" -# Only install BCC tracing packages in Cirrus CI. -if [[ "${CIRRUS_CI}" == "true" ]]; then + +# Only install BCC tracing packages in CI. Container has to match the host for BCC to work. +if [[ "${INSTALL_BCC_TRACING_TOOLS}" == "true" ]]; then + # Required for USDT functional tests to run BPFCC_PACKAGE="bpfcc-tools linux-headers-$(uname --kernel-release)" export CI_CONTAINER_CAP="--privileged -v /sys/kernel:/sys/kernel:rw" else @@ -24,3 +26,4 @@ export BITCOIN_CONFIG="--enable-usdt --enable-zmq --with-incompatible-bdb --with CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' \ --with-sanitizers=address,float-divide-by-zero,integer,undefined \ CC='clang-18 -ftrivial-auto-var-init=pattern' CXX='clang++-18 -ftrivial-auto-var-init=pattern'" +export CCACHE_MAXSIZE=300M diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index 25962a53e5..f16321ba55 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -86,14 +86,4 @@ if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ] tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" fi -if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then - ANDROID_TOOLS_PATH=${DEPENDS_DIR}/sdk-sources/android-tools.zip - if [ ! -f "$ANDROID_TOOLS_PATH" ]; then - ${CI_RETRY_EXE} curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH" - fi - mkdir -p "$ANDROID_HOME" - unzip -o "$ANDROID_TOOLS_PATH" -d "$ANDROID_HOME" - yes | "${ANDROID_HOME}"/cmdline-tools/bin/sdkmanager --sdk_root="${ANDROID_HOME}" --install "build-tools;${ANDROID_BUILD_TOOLS_VERSION}" "platform-tools" "platforms;android-31" "platforms;android-${ANDROID_API_LEVEL}" "ndk;${ANDROID_NDK_VERSION}" -fi - git config --global ${CFG_DONE} "true" diff --git a/ci/test/02_run_container.sh b/ci/test/02_run_container.sh index e6c4a61341..afd447c347 100755 --- a/ci/test/02_run_container.sh +++ b/ci/test/02_run_container.sh @@ -16,6 +16,7 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then # System-dependent env vars must be kept as is. So read them from the container. docker run --rm "${CI_IMAGE_NAME_TAG}" bash -c "env | grep --extended-regexp '^(HOME|PATH|USER)='" | tee --append "/tmp/env-$USER-$CONTAINER_NAME" echo "Creating $CI_IMAGE_NAME_TAG container to run in" + DOCKER_BUILDKIT=1 docker build \ --file "${BASE_READ_ONLY_DIR}/ci/test_imagefile" \ --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \ @@ -23,12 +24,32 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then --label="${CI_IMAGE_LABEL}" \ --tag="${CONTAINER_NAME}" \ "${BASE_READ_ONLY_DIR}" + docker volume create "${CONTAINER_NAME}_ccache" || true docker volume create "${CONTAINER_NAME}_depends" || true docker volume create "${CONTAINER_NAME}_depends_sources" || true - docker volume create "${CONTAINER_NAME}_depends_SDKs_android" || true docker volume create "${CONTAINER_NAME}_previous_releases" || true + CI_CCACHE_MOUNT="type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR" + CI_DEPENDS_MOUNT="type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR/built" + CI_DEPENDS_SOURCES_MOUNT="type=volume,src=${CONTAINER_NAME}_depends_sources,dst=$DEPENDS_DIR/sources" + CI_PREVIOUS_RELEASES_MOUNT="type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR" + + if [ "$DANGER_CI_ON_HOST_CACHE_FOLDERS" ]; then + # ensure the directories exist + mkdir -p "${CCACHE_DIR}" + mkdir -p "${DEPENDS_DIR}/built" + mkdir -p "${DEPENDS_DIR}/sources" + mkdir -p "${PREVIOUS_RELEASES_DIR}" + + CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=$CCACHE_DIR" + CI_DEPENDS_MOUNT="type=bind,src=${DEPENDS_DIR}/built,dst=$DEPENDS_DIR/built" + CI_DEPENDS_SOURCES_MOUNT="type=bind,src=${DEPENDS_DIR}/sources,dst=$DEPENDS_DIR/sources" + CI_PREVIOUS_RELEASES_MOUNT="type=bind,src=${PREVIOUS_RELEASES_DIR},dst=$PREVIOUS_RELEASES_DIR" + fi + + docker network create --ipv6 --subnet 1111:1111::/112 ci-ip6net || true + if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then echo "Restart docker before run to stop and clear all containers started with --rm" podman container rm --force --all # Similar to "systemctl restart docker" @@ -49,13 +70,13 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then # shellcheck disable=SC2086 CI_CONTAINER_ID=$(docker run --cap-add LINUX_IMMUTABLE $CI_CONTAINER_CAP --rm --interactive --detach --tty \ --mount "type=bind,src=$BASE_READ_ONLY_DIR,dst=$BASE_READ_ONLY_DIR,readonly" \ - --mount "type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR" \ - --mount "type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR/built" \ - --mount "type=volume,src=${CONTAINER_NAME}_depends_sources,dst=$DEPENDS_DIR/sources" \ - --mount "type=volume,src=${CONTAINER_NAME}_depends_SDKs_android,dst=$DEPENDS_DIR/SDKs/android" \ - --mount "type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR" \ + --mount "${CI_CCACHE_MOUNT}" \ + --mount "${CI_DEPENDS_MOUNT}" \ + --mount "${CI_DEPENDS_SOURCES_MOUNT}" \ + --mount "${CI_PREVIOUS_RELEASES_MOUNT}" \ --env-file /tmp/env-$USER-$CONTAINER_NAME \ --name "$CONTAINER_NAME" \ + --network ci-ip6net \ "$CONTAINER_NAME") export CI_CONTAINER_ID export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}" diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh index f5da7bc55d..71d9ad7f72 100755 --- a/ci/test/03_test_script.sh +++ b/ci/test/03_test_script.sh @@ -110,15 +110,6 @@ fi ccache --zero-stats PRINT_CCACHE_STATISTICS="ccache --version | head -n 1 && ccache --show-stats" -if [ -n "$ANDROID_TOOLS_URL" ]; then - make distclean || true - ./autogen.sh - bash -c "./configure $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG" || ( (cat config.log) && false) - make "${MAKEJOBS}" && cd src/qt && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk - bash -c "${PRINT_CCACHE_STATISTICS}" - exit 0 -fi - BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --prefix=$BASE_OUTDIR" if [ -n "$CONFIG_SHELL" ]; then diff --git a/configure.ac b/configure.ac index 6177324001..23b8870d43 100644 --- a/configure.ac +++ b/configure.ac @@ -206,9 +206,9 @@ AC_ARG_WITH([qrencode], AC_ARG_ENABLE([hardening], [AS_HELP_STRING([--disable-hardening], - [do not attempt to harden the resulting executables (default is to harden when possible)])], + [do not attempt to harden the resulting executables (default is to harden)])], [use_hardening=$enableval], - [use_hardening=auto]) + [use_hardening=yes]) AC_ARG_ENABLE([reduce-exports], [AS_HELP_STRING([--enable-reduce-exports], @@ -238,12 +238,6 @@ AC_ARG_ENABLE([lcov], [use_lcov=$enableval], [use_lcov=no]) -AC_ARG_ENABLE([lcov-branch-coverage], - [AS_HELP_STRING([--enable-lcov-branch-coverage], - [enable lcov testing branch coverage (default is no)])], - [use_lcov_branch=yes], - [use_lcov_branch=no]) - AC_ARG_ENABLE([zmq], [AS_HELP_STRING([--disable-zmq], [disable ZMQ notifications])], @@ -287,13 +281,6 @@ AC_ARG_WITH([sanitizers], [comma separated list of extra sanitizers to build with (default is none enabled)])], [use_sanitizers=$withval]) -dnl Enable gprof profiling -AC_ARG_ENABLE([gprof], - [AS_HELP_STRING([--enable-gprof], - [use gprof profiling compiler flags (default is no)])], - [enable_gprof=$enableval], - [enable_gprof=no]) - dnl Turn warnings into errors AC_ARG_ENABLE([werror], [AS_HELP_STRING([--enable-werror], @@ -417,15 +404,13 @@ AX_CHECK_COMPILE_FLAG([-Wsuggest-override], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsug AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wimplicit-fallthrough"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wunreachable-code], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunreachable-code"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wdocumentation], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdocumentation"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wself-assign], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wself-assign"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wundef], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wundef"], [], [$CXXFLAG_WERROR]) dnl Some compilers (gcc) ignore unknown -Wno-* options, but warn about all dnl unknown options if any other warning is produced. Test the -Wfoo case, and dnl set the -Wno-foo case if it works. AX_CHECK_COMPILE_FLAG([-Wunused-parameter], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-parameter"], [], [$CXXFLAG_WERROR]) -AX_CHECK_COMPILE_FLAG([-Wself-assign], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-self-assign"], [], [$CXXFLAG_WERROR]) -if test "$suppress_external_warnings" != "yes" ; then - AX_CHECK_COMPILE_FLAG([-Wdeprecated-copy], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-deprecated-copy"], [], [$CXXFLAG_WERROR]) -fi dnl Don't allow extended (non-ASCII) symbols in identifiers. This is easier for code review. AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers], [CORE_CXXFLAGS="$CORE_CXXFLAGS -fno-extended-identifiers"], [], [$CXXFLAG_WERROR]) @@ -772,22 +757,6 @@ case $host in dnl "'NSUserNotificationCenter' is deprecated: first deprecated in macOS 11.0". OBJCXXFLAGS="$CXXFLAGS -Wno-deprecated-declarations" ;; - *android*) - dnl make sure android stays above linux for hosts like *linux-android* - TARGET_OS=android - case $host in - *x86_64*) - ANDROID_ARCH=x86_64 - ;; - *aarch64*) - ANDROID_ARCH=arm64-v8a - ;; - *armv7a*) - ANDROID_ARCH=armeabi-v7a - ;; - *) AC_MSG_ERROR([Could not determine Android arch, or it is unsupported]) ;; - esac - ;; *linux*) TARGET_OS=linux ;; @@ -837,10 +806,8 @@ if test "$use_lcov" = "yes"; then AX_CHECK_COMPILE_FLAG([--coverage],[CORE_CXXFLAGS="$CORE_CXXFLAGS --coverage"], [AC_MSG_ERROR([lcov testing requested but --coverage flag does not work])]) CORE_CXXFLAGS="$CORE_CXXFLAGS -Og" -fi -if test "$use_lcov_branch" != "no"; then - AC_SUBST(LCOV_OPTS, "$LCOV_OPTS --rc lcov_branch_coverage=1") + AC_SUBST(LCOV_OPTS) fi dnl Check for endianness @@ -871,30 +838,12 @@ if test "$ac_cv_sys_large_files" != "" && CORE_CPPFLAGS="$CORE_CPPFLAGS -D_LARGE_FILES=$ac_cv_sys_large_files" fi -if test "$enable_gprof" = "yes"; then - dnl -pg is incompatible with -pie. Since hardening and profiling together doesn't make sense, - dnl we simply make them mutually exclusive here. Additionally, hardened toolchains may force - dnl -pie by default, in which case it needs to be turned off with -no-pie. - - if test "$use_hardening" = "yes"; then - AC_MSG_ERROR([gprof profiling is not compatible with hardening. Reconfigure with --disable-hardening or --disable-gprof]) - fi - use_hardening=no - AX_CHECK_COMPILE_FLAG([-pg],[GPROF_CXXFLAGS="-pg"], - [AC_MSG_ERROR([gprof profiling requested but not available])], [$CXXFLAG_WERROR]) - - AX_CHECK_LINK_FLAG([-no-pie], [GPROF_LDFLAGS="-no-pie"]) - AX_CHECK_LINK_FLAG([-pg], [GPROF_LDFLAGS="$GPROF_LDFLAGS -pg"], - [AC_MSG_ERROR([gprof profiling requested but not available])], [$GPROF_LDFLAGS]) -fi - if test "$TARGET_OS" != "windows"; then dnl All windows code is PIC, forcing it on just adds useless compile warnings AX_CHECK_COMPILE_FLAG([-fPIC], [PIC_FLAGS="-fPIC"]) fi if test "$use_hardening" != "no"; then - use_hardening=yes AX_CHECK_COMPILE_FLAG([-Wstack-protector], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -Wstack-protector"]) AX_CHECK_COMPILE_FLAG([-fstack-protector-all], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-protector-all"]) @@ -1444,6 +1393,9 @@ if test "$with_libmultiprocess" = "yes" || test "$with_libmultiprocess" = "auto" PKG_CHECK_MODULES([LIBMULTIPROCESS], [libmultiprocess], [ libmultiprocess_found=yes; libmultiprocess_prefix=`$PKG_CONFIG --variable=prefix libmultiprocess`; + if test "$suppress_external_warnings" != "no" ; then + LIBMULTIPROCESS_CFLAGS=SUPPRESS_WARNINGS($LIBMULTIPROCESS_CFLAGS) + fi ], [true]) elif test "$with_libmultiprocess" != "no"; then AC_MSG_ERROR([--with-libmultiprocess=$with_libmultiprocess value is not yes, auto, or no]) @@ -1711,8 +1663,6 @@ AC_SUBST(WARN_CXXFLAGS) AC_SUBST(NOWARN_CXXFLAGS) AC_SUBST(DEBUG_CXXFLAGS) AC_SUBST(ERROR_CXXFLAGS) -AC_SUBST(GPROF_CXXFLAGS) -AC_SUBST(GPROF_LDFLAGS) AC_SUBST(HARDENED_CXXFLAGS) AC_SUBST(HARDENED_CPPFLAGS) AC_SUBST(HARDENED_LDFLAGS) @@ -1744,7 +1694,6 @@ AC_SUBST(HAVE_O_CLOEXEC) AC_SUBST(HAVE_BUILTIN_PREFETCH) AC_SUBST(HAVE_MM_PREFETCH) AC_SUBST(HAVE_STRONG_GETAUXVAL) -AC_SUBST(ANDROID_ARCH) AC_SUBST(HAVE_EVHTTP_CONNECTION_GET_PEER_CONST_CHAR) AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/config.ini]) AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh]) @@ -1821,7 +1770,6 @@ echo " with natpmp = $use_natpmp" echo " USDT tracing = $use_usdt" echo " sanitizers = $use_sanitizers" echo " debug enabled = $enable_debug" -echo " gprof enabled = $enable_gprof" echo " werror = $enable_werror" echo echo " target os = $host_os" @@ -1831,8 +1779,8 @@ echo " CC = $CC" echo " CFLAGS = $PTHREAD_CFLAGS $SANITIZER_CFLAGS $CFLAGS" echo " CPPFLAGS = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CORE_CPPFLAGS $CPPFLAGS" echo " CXX = $CXX" -echo " CXXFLAGS = $CORE_CXXFLAGS $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $SANITIZER_CXXFLAGS $CXXFLAGS" -echo " LDFLAGS = $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $SANITIZER_LDFLAGS $CORE_LDFLAGS $LDFLAGS" +echo " CXXFLAGS = $CORE_CXXFLAGS $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $SANITIZER_CXXFLAGS $CXXFLAGS" +echo " LDFLAGS = $PTHREAD_LIBS $HARDENED_LDFLAGS $SANITIZER_LDFLAGS $CORE_LDFLAGS $LDFLAGS" echo " AR = $AR" echo " ARFLAGS = $ARFLAGS" echo diff --git a/contrib/devtools/check-deps.sh b/contrib/devtools/check-deps.sh new file mode 100755 index 0000000000..9d2eebe14d --- /dev/null +++ b/contrib/devtools/check-deps.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash + +export LC_ALL=C +set -Eeuo pipefail + +# Declare paths to libraries +declare -A LIBS +LIBS[cli]="libbitcoin_cli.a" +LIBS[common]="libbitcoin_common.a" +LIBS[consensus]="libbitcoin_consensus.a" +LIBS[crypto]="crypto/.libs/libbitcoin_crypto_base.a crypto/.libs/libbitcoin_crypto_x86_shani.a crypto/.libs/libbitcoin_crypto_sse41.a crypto/.libs/libbitcoin_crypto_avx2.a" +LIBS[node]="libbitcoin_node.a" +LIBS[util]="libbitcoin_util.a" +LIBS[wallet]="libbitcoin_wallet.a" +LIBS[wallet_tool]="libbitcoin_wallet_tool.a" + +# Declare allowed dependencies "X Y" where X is allowed to depend on Y. This +# list is taken from doc/design/libraries.md. +ALLOWED_DEPENDENCIES=( + "cli common" + "cli util" + "common consensus" + "common crypto" + "common util" + "consensus crypto" + "node common" + "node consensus" + "node crypto" + "node kernel" + "node util" + "util crypto" + "wallet common" + "wallet crypto" + "wallet util" + "wallet_tool util" + "wallet_tool wallet" +) + +# Add minor dependencies omitted from doc/design/libraries.md to keep the +# dependency diagram simple. +ALLOWED_DEPENDENCIES+=( + "wallet consensus" + "wallet_tool common" + "wallet_tool crypto" +) + +# Declare list of known errors that should be suppressed. +declare -A SUPPRESS +# init.cpp file currently calls Berkeley DB sanity check function on startup, so +# there is an undocumented dependency of the node library on the wallet library. +SUPPRESS["libbitcoin_node_a-init.o libbitcoin_wallet_a-bdb.o _ZN6wallet27BerkeleyDatabaseSanityCheckEv"]=1 +# init/common.cpp file calls InitError and InitWarning from interface_ui which +# is currently part of the node library. interface_ui should just be part of the +# common library instead, and is moved in +# https://github.com/bitcoin/bitcoin/issues/10102 +SUPPRESS["libbitcoin_common_a-common.o libbitcoin_node_a-interface_ui.o _Z11InitWarningRK13bilingual_str"]=1 +SUPPRESS["libbitcoin_common_a-common.o libbitcoin_node_a-interface_ui.o _Z9InitErrorRK13bilingual_str"]=1 +# rpc/external_signer.cpp adds defines node RPC methods but is built as part of the +# common library. It should be moved to the node library instead. +SUPPRESS["libbitcoin_common_a-external_signer.o libbitcoin_node_a-server.o _ZN9CRPCTable13appendCommandERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPK11CRPCCommand"]=1 + +usage() { + echo "Usage: $(basename "${BASH_SOURCE[0]}") [BUILD_DIR]" +} + +# Output makefile targets, converting library .a paths to libtool .la targets +lib_targets() { + for lib in "${!LIBS[@]}"; do + for lib_path in ${LIBS[$lib]}; do + # shellcheck disable=SC2001 + sed 's:/.libs/\(.*\)\.a$:/\1.la:g' <<<"$lib_path" + done + done +} + +# Extract symbol names and object names and write to text files +extract_symbols() { + local temp_dir="$1" + for lib in "${!LIBS[@]}"; do + for lib_path in ${LIBS[$lib]}; do + nm -o "$lib_path" | grep ' T ' | awk '{print $3, $1}' >> "${temp_dir}/${lib}_exports.txt" + nm -o "$lib_path" | grep ' U ' | awk '{print $3, $1}' >> "${temp_dir}/${lib}_imports.txt" + awk '{print $1}' "${temp_dir}/${lib}_exports.txt" | sort -u > "${temp_dir}/${lib}_exported_symbols.txt" + awk '{print $1}' "${temp_dir}/${lib}_imports.txt" | sort -u > "${temp_dir}/${lib}_imported_symbols.txt" + done + done +} + +# Lookup object name(s) corresponding to symbol name in text file +obj_names() { + local symbol="$1" + local txt_file="$2" + sed -n "s/^$symbol [^:]\\+:\\([^:]\\+\\):[^:]*\$/\\1/p" "$txt_file" | sort -u +} + +# Iterate through libraries and find disallowed dependencies +check_libraries() { + local temp_dir="$1" + local result=0 + for src in "${!LIBS[@]}"; do + for dst in "${!LIBS[@]}"; do + if [ "$src" != "$dst" ] && ! is_allowed "$src" "$dst"; then + if ! check_disallowed "$src" "$dst" "$temp_dir"; then + result=1 + fi + fi + done + done + check_not_suppressed + return $result +} + +# Return whether src library is allowed to depend on dst. +is_allowed() { + local src="$1" + local dst="$2" + for allowed in "${ALLOWED_DEPENDENCIES[@]}"; do + if [ "$src $dst" = "$allowed" ]; then + return 0 + fi + done + return 1 +} + +# Return whether src library imports any symbols from dst, assuming src is not +# allowed to depend on dst. +check_disallowed() { + local src="$1" + local dst="$2" + local temp_dir="$3" + local result=0 + + # Loop over symbol names exported by dst and imported by src + while read symbol; do + local dst_obj + dst_obj=$(obj_names "$symbol" "${temp_dir}/${dst}_exports.txt") + while read src_obj; do + if ! check_suppress "$src_obj" "$dst_obj" "$symbol"; then + echo "Error: $src_obj depends on $dst_obj symbol '$(c++filt "$symbol")', can suppess with:" + echo " SUPPRESS[\"$src_obj $dst_obj $symbol\"]=1" + result=1 + fi + done < <(obj_names "$symbol" "${temp_dir}/${src}_imports.txt") + done < <(comm -12 "${temp_dir}/${dst}_exported_symbols.txt" "${temp_dir}/${src}_imported_symbols.txt") + return $result +} + +# Declare array to track errors which were suppressed. +declare -A SUPPRESSED + +# Return whether error should be suppressed and record suppresssion in +# SUPPRESSED array. +check_suppress() { + local src_obj="$1" + local dst_obj="$2" + local symbol="$3" + for suppress in "${!SUPPRESS[@]}"; do + read suppress_src suppress_dst suppress_pattern <<<"$suppress" + if [[ "$src_obj" == "$suppress_src" && "$dst_obj" == "$suppress_dst" && "$symbol" =~ $suppress_pattern ]]; then + SUPPRESSED["$suppress"]=1 + return 0 + fi + done + return 1 +} + +# Warn about error which were supposed to be suppress, but were not encountered. +check_not_suppressed() { + for suppress in "${!SUPPRESS[@]}"; do + if [[ ! -v SUPPRESSED[$suppress] ]]; then + echo >&2 "Warning: suppression '$suppress' was ignored, consider deleting." + fi + done +} + +# Check arguments. +if [ "$#" = 0 ]; then + BUILD_DIR="$(dirname "${BASH_SOURCE[0]}")/../../src" +elif [ "$#" = 1 ]; then + BUILD_DIR="$1" +else + echo >&2 "Error: wrong number of arguments." + usage >&2 + exit 1 +fi +if [ ! -f "$BUILD_DIR/Makefile" ]; then + echo >&2 "Error: directory '$BUILD_DIR' does not contain a makefile, please specify path to build directory for library targets." + usage >&2 + exit 1 +fi + +# Build libraries and run checks. +cd "$BUILD_DIR" +# shellcheck disable=SC2046 +make -j"$(nproc)" $(lib_targets) +TEMP_DIR="$(mktemp -d)" +extract_symbols "$TEMP_DIR" +if check_libraries "$TEMP_DIR"; then + echo "Success! No unexpected dependencies were detected." +else + echo >&2 "Error: Unexpected dependencies were detected. Check previous output." +fi +rm -r "$TEMP_DIR" diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index b3e73bb2b9..1d487d4f6b 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -212,6 +212,11 @@ def check_exported_symbols(binary) -> bool: ok = False return ok +def check_RUNPATH(binary) -> bool: + assert binary.get(lief.ELF.DYNAMIC_TAGS.RUNPATH) is None + assert binary.get(lief.ELF.DYNAMIC_TAGS.RPATH) is None + return True + def check_ELF_libraries(binary) -> bool: ok: bool = True for library in binary.libraries: @@ -239,8 +244,8 @@ def check_MACHO_sdk(binary) -> bool: return True return False -def check_MACHO_ld64(binary) -> bool: - if binary.build_version.tools[0].version == [711, 0, 0]: +def check_MACHO_lld(binary) -> bool: + if binary.build_version.tools[0].version == [18, 1, 6]: return True return False @@ -277,12 +282,13 @@ lief.EXE_FORMATS.ELF: [ ('LIBRARY_DEPENDENCIES', check_ELF_libraries), ('INTERPRETER_NAME', check_ELF_interpreter), ('ABI', check_ELF_ABI), + ('RUNPATH', check_RUNPATH), ], lief.EXE_FORMATS.MACHO: [ ('DYNAMIC_LIBRARIES', check_MACHO_libraries), ('MIN_OS', check_MACHO_min_os), ('SDK', check_MACHO_sdk), - ('LD64', check_MACHO_ld64), + ('LLD', check_MACHO_lld), ], lief.EXE_FORMATS.PE: [ ('DYNAMIC_LIBRARIES', check_PE_libraries), diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 51bca4627e..dd0cf7030a 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -120,21 +120,15 @@ class TestSecurityChecks(unittest.TestCase): arch = get_arch(cc, source, executable) if arch == lief.ARCHITECTURES.X86: - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector', '-Wl,-no_fixup_chains']), - (1, executable+': failed NOUNDEFS Canary FIXUP_CHAINS PIE NX CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector', '-Wl,-fixup_chains']), - (1, executable+': failed NOUNDEFS Canary PIE NX CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all', '-Wl,-fixup_chains']), - (1, executable+': failed NOUNDEFS PIE NX CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all', '-Wl,-fixup_chains']), - (1, executable+': failed NOUNDEFS PIE CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all', '-Wl,-fixup_chains']), - (1, executable+': failed PIE CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all', '-Wl,-fixup_chains']), - (1, executable+': failed PIE CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all', '-fcf-protection=full', '-Wl,-fixup_chains']), - (1, executable+': failed PIE')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-fstack-protector-all', '-fcf-protection=full', '-Wl,-fixup_chains']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fno-stack-protector', '-Wl,-no_fixup_chains']), + (1, executable+': failed NOUNDEFS Canary FIXUP_CHAINS PIE CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector', '-Wl,-fixup_chains']), + (1, executable+': failed NOUNDEFS Canary CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all', '-Wl,-fixup_chains']), + (1, executable+': failed NOUNDEFS CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all', '-Wl,-fixup_chains']), + (1, executable+': failed CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all', '-fcf-protection=full', '-Wl,-fixup_chains']), (0, '')) else: # arm64 darwin doesn't support non-PIE binaries, control flow or executable stacks diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index 870938cb52..2ea574fe4b 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -361,6 +361,10 @@ INFO: Building ${VERSION:?not set} for platform triple ${HOST:?not set}: ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' ...outputting in: '$(outdir_for_host "$HOST")' ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")' + ADDITIONAL FLAGS (if set) + ADDITIONAL_GUIX_COMMON_FLAGS: ${ADDITIONAL_GUIX_COMMON_FLAGS} + ADDITIONAL_GUIX_ENVIRONMENT_FLAGS: ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} + ADDITIONAL_GUIX_TIMEMACHINE_FLAGS: ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} EOF # Run the build script 'contrib/guix/libexec/build.sh' in the build diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 1e9b682f3f..9bc8c0e75d 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -133,18 +133,7 @@ for p in "${PATHS[@]}"; do done # Disable Guix ld auto-rpath behavior -case "$HOST" in - *darwin*) - # The auto-rpath behavior is necessary for darwin builds as some native - # tools built by depends refer to and depend on Guix-built native - # libraries - # - # After the native packages in depends are built, the ld wrapper should - # no longer affect our build, as clang would instead reach for - # x86_64-apple-darwin-ld from cctools - ;; - *) export GUIX_LD_WRAPPER_DISABLE_RPATH=yes ;; -esac +export GUIX_LD_WRAPPER_DISABLE_RPATH=yes # Make /usr/bin if it doesn't exist [ -e /usr/bin ] || mkdir -p /usr/bin @@ -173,16 +162,6 @@ esac # Environment variables for determinism export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" export TZ="UTC" -case "$HOST" in - *darwin*) - # cctools AR, unlike GNU binutils AR, does not have a deterministic mode - # or a configure flag to enable determinism by default, it only - # understands if this env-var is set or not. See: - # - # https://github.com/tpoechtrager/cctools-port/blob/55562e4073dea0fbfd0b20e0bf69ffe6390c7f97/cctools/ar/archive.c#L334 - export ZERO_AR_DATE=yes - ;; -esac #################### # Depends Building # @@ -199,8 +178,7 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \ x86_64_linux_AR=x86_64-linux-gnu-gcc-ar \ x86_64_linux_RANLIB=x86_64-linux-gnu-gcc-ranlib \ x86_64_linux_NM=x86_64-linux-gnu-gcc-nm \ - x86_64_linux_STRIP=x86_64-linux-gnu-strip \ - FORCE_USE_SYSTEM_CLANG=1 + x86_64_linux_STRIP=x86_64-linux-gnu-strip ########################### diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index ce6a9562b4..80bfb2875f 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -51,7 +51,7 @@ fi time-machine() { # shellcheck disable=SC2086 guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=dc4842797bfdc5f9f3f5f725bf189c2b68bd6b5a \ + --commit=f0bb724211872cd6158fce6162e0b8c73efed126 \ --cores="$JOBS" \ --keep-failed \ --fallback \ diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 96818c7748..53569d7f7d 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -532,8 +532,9 @@ inspecting signatures in Mach-O binaries.") ((string-contains target "darwin") (list ;; Native GCC 11 toolchain gcc-toolchain-11 - binutils - clang-toolchain-17 + clang-toolchain-18 + lld-18 + (make-lld-wrapper lld-18 #:lld-as-ld? #t) python-signapple zip)) (else '()))))) diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index d1df3062f8..d47ee6774e 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -56,45 +56,22 @@ The `sha256sum` should be `c0c2e7bb92c1fee0c4e9f3a485e4530786732d6c6dd9e9f418c28 ## Deterministic macOS App Notes -macOS Applications are created in Linux by combining a recent `clang` and the Apple -`binutils` (`ld`, `ar`, etc). +macOS Applications are created on Linux using a recent LLVM. -Apple uses `clang` extensively for development and has upstreamed the necessary -functionality so that a vanilla clang can take advantage. It supports the use of `-F`, -`-target`, `-mmacosx-version-min`, and `-isysroot`, which are all necessary when -building for macOS. +All builds must target an Apple SDK. These SDKs are free to download, but not redistributable. +See the SDK Extraction notes above for how to obtain it. -Apple's version of `binutils` (called `cctools`) contains lots of functionality missing in the -FSF's `binutils`. In addition to extra linker options for frameworks and sysroots, several -other tools are needed as well. These do not build under Linux, so they have been patched to -do so. The work here was used as a starting point: [mingwandroid/toolchain4](https://github.com/mingwandroid/toolchain4). +The Guix build process has been designed to avoid including the SDK's files in Guix's outputs. +All interim tarballs are fully deterministic and may be freely redistributed. -In order to build a working toolchain, the following source packages are needed from -Apple: `cctools`, `dyld`, and `ld64`. - -These tools inject timestamps by default, which produce non-deterministic binaries. The -`ZERO_AR_DATE` environment variable is used to disable that. - -This version of `cctools` has been patched to use the current version of `clang`'s headers -and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done in `toolchain4`. - -To complicate things further, all builds must target an Apple SDK. These SDKs are free to -download, but not redistributable. See the SDK Extraction notes above for how to obtain it. - -The Guix process builds 2 sets of files: Linux tools, then Apple binaries which are -created using these tools. The build process has been designed to avoid including the -SDK's files in Guix's outputs. All interim tarballs are fully deterministic and may be freely -redistributed. - -As of OS X 10.9 Mavericks, using an Apple-blessed key to sign binaries is a requirement in -order to satisfy the new Gatekeeper requirements. Because this private key cannot be -shared, we'll have to be a bit creative in order for the build process to remain somewhat -deterministic. Here's how it works: +Using an Apple-blessed key to sign binaries is a requirement to produce (distributable) macOS +binaries. Because this private key cannot be shared, we'll have to be a bit creative in order +for the build process to remain somewhat deterministic. Here's how it works: - Builders use Guix to create an unsigned release. This outputs an unsigned ZIP which - users may choose to bless and run. It also outputs an unsigned app structure in the form - of a tarball. + users may choose to bless, self-codesign, and run. It also outputs an unsigned app structure + in the form of a tarball. - The Apple keyholder uses this unsigned app to create a detached signature, using the - script that is also included there. Detached signatures are available from this [repository](https://github.com/bitcoin-core/bitcoin-detached-sigs). -- Builders feed the unsigned app + detached signature back into Guix. It uses the - pre-built tools to recombine the pieces into a deterministic ZIP. + included script. Detached signatures are available from this [repository](https://github.com/bitcoin-core/bitcoin-detached-sigs). +- Builders feed the unsigned app + detached signature back into Guix, which combines the + pieces into a deterministic ZIP. diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk index b73f5cba14..86a6262b5c 100755 --- a/contrib/macdeploy/gen-sdk +++ b/contrib/macdeploy/gen-sdk @@ -8,21 +8,6 @@ import gzip import os import contextlib -# monkey-patch Python 3.8 and older to fix wrong TAR header handling -# see https://github.com/bitcoin/bitcoin/pull/24534 -# and https://github.com/python/cpython/pull/18080 for more info -if sys.version_info < (3, 9): - _old_create_header = tarfile.TarInfo._create_header - def _create_header(info, format, encoding, errors): - buf = _old_create_header(info, format, encoding, errors) - # replace devmajor/devminor with binary zeroes - buf = buf[:329] + bytes(16) + buf[345:] - # recompute checksum - chksum = tarfile.calc_chksums(buf)[0] - buf = buf[:-364] + bytes("%06o\0" % chksum, "ascii") + buf[-357:] - return buf - tarfile.TarInfo._create_header = staticmethod(_create_header) - @contextlib.contextmanager def cd(path): """Context manager that restores PWD even if an exception was raised.""" diff --git a/contrib/seeds/generate-seeds.py b/contrib/seeds/generate-seeds.py index e921757802..f67e7b0f4c 100755 --- a/contrib/seeds/generate-seeds.py +++ b/contrib/seeds/generate-seeds.py @@ -29,7 +29,6 @@ These should be pasted into `src/chainparamsseeds.h`. from base64 import b32decode from enum import Enum -import struct import sys import os import re @@ -115,13 +114,13 @@ def parse_spec(s): def ser_compact_size(l): r = b"" if l < 253: - r = struct.pack("B", l) + r = l.to_bytes(1, "little") elif l < 0x10000: - r = struct.pack("<BH", 253, l) + r = (253).to_bytes(1, "little") + l.to_bytes(2, "little") elif l < 0x100000000: - r = struct.pack("<BI", 254, l) + r = (254).to_bytes(1, "little") + l.to_bytes(4, "little") else: - r = struct.pack("<BQ", 255, l) + r = (255).to_bytes(1, "little") + l.to_bytes(8, "little") return r def bip155_serialize(spec): @@ -129,10 +128,10 @@ def bip155_serialize(spec): Serialize (networkID, addr, port) tuple to BIP155 binary format. ''' r = b"" - r += struct.pack('B', spec[0].value) + r += spec[0].value.to_bytes(1, "little") r += ser_compact_size(len(spec[1])) r += spec[1] - r += struct.pack('>H', spec[2]) + r += spec[2].to_bytes(2, "big") return r def process_nodes(g, f, structname): diff --git a/contrib/signet/miner b/contrib/signet/miner index e5daf9f993..4216ada5fa 100755 --- a/contrib/signet/miner +++ b/contrib/signet/miner @@ -52,20 +52,20 @@ def signet_txs(block, challenge): mroot = block.get_merkle_root(hashes) sd = b"" - sd += struct.pack("<i", block.nVersion) + sd += block.nVersion.to_bytes(4, "little", signed=True) sd += ser_uint256(block.hashPrevBlock) sd += ser_uint256(mroot) - sd += struct.pack("<I", block.nTime) + sd += block.nTime.to_bytes(4, "little") to_spend = CTransaction() - to_spend.nVersion = 0 + to_spend.version = 0 to_spend.nLockTime = 0 to_spend.vin = [CTxIn(COutPoint(0, 0xFFFFFFFF), b"\x00" + CScriptOp.encode_op_pushdata(sd), 0)] to_spend.vout = [CTxOut(0, challenge)] to_spend.rehash() spend = CTransaction() - spend.nVersion = 0 + spend.version = 0 spend.nLockTime = 0 spend.vin = [CTxIn(COutPoint(to_spend.sha256, 0), b"", 0)] spend.vout = [CTxOut(0, b"\x6a")] diff --git a/contrib/windeploy/win-codesign.cert b/contrib/windeploy/win-codesign.cert index 22f17296b6..ec60a3018d 100644 --- a/contrib/windeploy/win-codesign.cert +++ b/contrib/windeploy/win-codesign.cert @@ -1,8 +1,8 @@ -----BEGIN CERTIFICATE-----
-MIIHfDCCBWSgAwIBAgIQCmVvdQal72U2QxbUTT3SRTANBgkqhkiG9w0BAQsFADBp
+MIIHeTCCBWGgAwIBAgIQBzR46J2yq3g++NbQS/BBVDANBgkqhkiG9w0BAQsFADBp
MQswCQYDVQQGEwJVUzEXMBUGA1UEChMORGlnaUNlcnQsIEluYy4xQTA/BgNVBAMT
OERpZ2lDZXJ0IFRydXN0ZWQgRzQgQ29kZSBTaWduaW5nIFJTQTQwOTYgU0hBMzg0
-IDIwMjEgQ0ExMB4XDTIyMDUyNDAwMDAwMFoXDTI0MDUyOTIzNTk1OVowgYAxCzAJ
+IDIwMjEgQ0ExMB4XDTI0MDUyMjAwMDAwMFoXDTI3MDUzMTIzNTk1OVowgYAxCzAJ
BgNVBAYTAlVTMREwDwYDVQQIEwhEZWxhd2FyZTEOMAwGA1UEBxMFTGV3ZXMxJjAk
BgNVBAoTHUJpdGNvaW4gQ29yZSBDb2RlIFNpZ25pbmcgTExDMSYwJAYDVQQDEx1C
aXRjb2luIENvcmUgQ29kZSBTaWduaW5nIExMQzCCAiIwDQYJKoZIhvcNAQEBBQAD
@@ -17,28 +17,28 @@ CDvScIgnQXmk+cbKMBtg9kM0F+aLWsN2xVf0uAj3U7sdXLrfJeW0DZIktWtTBQzX O/OE4Ka+1WFnDg0HJIih0cTjl9YYvfe53L4pCGy+qGt/XGBRqCMfXp3g+H9FGR5r
pensVVcsrv3GbTfYdlpdmp9OHH5G57GTAZueobCZg7r7RKK0zPU9EiTLJxzyXuai
v/Ksd8eIhHRjewMaQuAtQM1tO+oKAbLF0v2M7v7/aVT76X32JllYAizm3zjvAgMB
-AAGjggIGMIICAjAfBgNVHSMEGDAWgBRoN+Drtjv4XxGG+/5hewiIZfROQjAdBgNV
-HQ4EFgQUvCpU58PIuofv0kHJ3Ty0YDKEy3cwDgYDVR0PAQH/BAQDAgeAMBMGA1Ud
-JQQMMAoGCCsGAQUFBwMDMIG1BgNVHR8Ega0wgaowU6BRoE+GTWh0dHA6Ly9jcmwz
-LmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydFRydXN0ZWRHNENvZGVTaWduaW5nUlNBNDA5
-NlNIQTM4NDIwMjFDQTEuY3JsMFOgUaBPhk1odHRwOi8vY3JsNC5kaWdpY2VydC5j
-b20vRGlnaUNlcnRUcnVzdGVkRzRDb2RlU2lnbmluZ1JTQTQwOTZTSEEzODQyMDIx
-Q0ExLmNybDA+BgNVHSAENzA1MDMGBmeBDAEEATApMCcGCCsGAQUFBwIBFhtodHRw
-Oi8vd3d3LmRpZ2ljZXJ0LmNvbS9DUFMwgZQGCCsGAQUFBwEBBIGHMIGEMCQGCCsG
+AAGjggIDMIIB/zAfBgNVHSMEGDAWgBRoN+Drtjv4XxGG+/5hewiIZfROQjAdBgNV
+HQ4EFgQUvCpU58PIuofv0kHJ3Ty0YDKEy3cwPgYDVR0gBDcwNTAzBgZngQwBBAEw
+KTAnBggrBgEFBQcCARYbaHR0cDovL3d3dy5kaWdpY2VydC5jb20vQ1BTMA4GA1Ud
+DwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzCBtQYDVR0fBIGtMIGqMFOg
+UaBPhk1odHRwOi8vY3JsMy5kaWdpY2VydC5jb20vRGlnaUNlcnRUcnVzdGVkRzRD
+b2RlU2lnbmluZ1JTQTQwOTZTSEEzODQyMDIxQ0ExLmNybDBToFGgT4ZNaHR0cDov
+L2NybDQuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0VHJ1c3RlZEc0Q29kZVNpZ25pbmdS
+U0E0MDk2U0hBMzg0MjAyMUNBMS5jcmwwgZQGCCsGAQUFBwEBBIGHMIGEMCQGCCsG
AQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20wXAYIKwYBBQUHMAKGUGh0
dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydFRydXN0ZWRHNENvZGVT
-aWduaW5nUlNBNDA5NlNIQTM4NDIwMjFDQTEuY3J0MAwGA1UdEwEB/wQCMAAwDQYJ
-KoZIhvcNAQELBQADggIBABhpTZufRws1vrtI0xB1/UWrSEJxdPHivfpXE708dzum
-Jh3TFzpsEUCQX5BJJet1l7x92sKNeAL7votA+8O8YvMD64Kim7VKA2BB8AOHKQbp
-r1c2iZBwwofInviRYvsrvQta6KBy2KOe1L/l0KnpUazL9Tv4VKvuWAw/Qc0/eTQr
-NZRsmADORxnZ1qW+SpF+/WbazIYjod/Oqb1U3on+PzyiGD3SjzNhsdFRptqzrIaY
-UVV+2XHG4fN6A8wkyQL5NIVXGiK7rqS5VrRAv58Lf1ZZTghdAL+5SySE0OsR9t0K
-W73ZB9pxbuZZ6Zfxjotjw+IilCEm3ADbc7Eb2ijI4x8mix0XWMUrhL34s7/jRyDi
-P+30aSgjWp611tp/EYRW5kpIaFR8AesDdM0DSSCCRXOMwQG2Tq2+CnqItB5oLNPp
-2XySwlIWvmjbzsREfIpE3yh3bxmHY+vFIc2R0nNkbWNIT6AGtaEQ7oWkgpK8YMkA
-QCf4EUC4Qa7qHiH6YSmYJhjApBLC7UDwevgwxuDrwimWAj+tDkzdnENMcBp4SAy6
-LwUuDi2IU6HRSXWdh2YEkDbc3FdwknnnEWaB4dlRL85YjHyLXN0KiE7SKTj1LfR4
-dGeDqVUlDj9D5+X4a7F89wLP/um40/52HUQv5t5WcNr/47r9aVkx9DHs1b8oUnLg
+aWduaW5nUlNBNDA5NlNIQTM4NDIwMjFDQTEuY3J0MAkGA1UdEwQCMAAwDQYJKoZI
+hvcNAQELBQADggIBADdniG9IY9oOfw1e3+uc2lR4hoZqquJQRrCnbWJ1npnCTavI
+CfcEEMuQ5ztg4TR7tQNj2KcaHWKuPYxEz2bg8HpSPG27lnXaz4pLgfqvjdZWNH2v
+W6DGRUAwuMQHSV0qhuRcJPZuhwSFx/8y4r++jIcBxCbt/Jprt/bqc8vZZZzTDPfG
+M6cGaKMDvF//OkUPVzh4s557kV7+LoaX8CigiACZky3Zj3tkQfJYkEvdQseNvX49
+CMJ+cjN+fGweshbn/DszAT5oXW5l2PXeceyGrE+5Ex1ELXCPqNj8ZSn+S9IKZOag
+zDFBA93RTVD438peXPz//xgusgnmSqSPS5tCp9KSvew81acu4v/+egg9EgSSx5Ho
+9fkOX7JuygvN3r3UZqsddxdwf2dPvBDYlMdieF8qsR7H5DQPQoaTVrIhW4TFtJl/
+UPjVlnDwu+yvMC70F+CaVgQs01uZ0VKuG3KNkkEj6+V/SM54NVVgcY/Q7llKIFA8
+Qk8Ip8/83cVBptKW+OU+i2ZwoYskLbdfDE31X2knUIouNZgBBMhzc5WjJCEGXAPm
+9xYZMn87cc+ejxCw6/WC4b6tDCziO8drq76Pl6LTNPOtRkEVqt12p8Uqi9PgznUB
+bdHeoF5XHt1Ca2ySpSYuMz5djwIC2ws8kiMm44/AyTm6dwRcesiOTqnaRc+t
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE----- MIIGsDCCBJigAwIBAgIQCK1AsmDSnEyfXs2pvZOu2TANBgkqhkiG9w0BAQwFADBi diff --git a/depends/Makefile b/depends/Makefile index 016c8f64ad..52a9a14e56 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -101,10 +101,6 @@ host_os+=$(findstring netbsd,$(full_host_os)) host_os+=$(findstring openbsd,$(full_host_os)) host_os+=$(findstring mingw32,$(full_host_os)) -ifeq (android,$(findstring android,$(full_host_os))) -host_os:=android -endif - host_os:=$(strip $(host_os)) ifeq ($(host_os),) host_os=$(full_host_os) @@ -185,9 +181,6 @@ all_packages = $(packages) $(native_packages) meta_depends = Makefile config.guess config.sub funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk -$(host_arch)_$(host_os)_native_binutils?=$($(host_os)_native_binutils) -$(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain) - include funcs.mk final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in) @@ -217,9 +210,8 @@ $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) # tool needs to be available in $PATH at all times. # # 2. If the tool is _**not**_ expected to be available in $PATH at all times -# (such as is the case for our native_cctools binutils tools), it needs to -# be referred to by its absolute path, such as would be output by the -# AC_PATH_{PROG,TOOL} macros. +# it needs to be referred to by its absolute path, such as would be output +# by the AC_PATH_{PROG,TOOL} macros. # # Minor note: it is also okay to refer to tools by their absolute path even if # we expect them to be available in $PATH at all times, more specificity does diff --git a/depends/README.md b/depends/README.md index 10e0985cf4..adaeed468f 100644 --- a/depends/README.md +++ b/depends/README.md @@ -38,21 +38,22 @@ Common `host-platform-triplet`s for cross compilation are: - `riscv32-linux-gnu` for Linux RISC-V 32 bit - `riscv64-linux-gnu` for Linux RISC-V 64 bit - `s390x-linux-gnu` for Linux S390X -- `armv7a-linux-android` for Android ARM 32 bit -- `aarch64-linux-android` for Android ARM 64 bit -- `x86_64-linux-android` for Android x86 64 bit -The paths are automatically configured and no other options are needed unless targeting [Android](../doc/build-android.md). +The paths are automatically configured and no other options are needed. ### Install the required dependencies: Ubuntu & Debian +#### Common + + apt install automake bison cmake curl libtool make patch pkg-config python3 xz-utils + #### For macOS cross compilation - sudo apt-get install curl bsdmainutils cmake zip + apt install clang lld llvm g++ zip -Note: You must obtain the macOS SDK before proceeding with a cross-compile. -Under the depends directory, create a subdirectory named `SDKs`. -Then, place the extracted SDK under this new directory. +Clang 18 or later is required. You must also obtain the macOS SDK before +proceeding with a cross-compile. Under the depends directory, create a +subdirectory named `SDKs`. Then, place the extracted SDK under this new directory. For more information, see [SDK Extraction](../contrib/macdeploy/README.md#sdk-extraction). #### For Win64 cross compilation @@ -63,7 +64,7 @@ For more information, see [SDK Extraction](../contrib/macdeploy/README.md#sdk-ex Common linux dependencies: - sudo apt-get install make automake cmake curl g++-multilib libtool binutils bsdmainutils pkg-config python3 patch bison + sudo apt-get install g++-multilib binutils For linux ARM cross compilation: @@ -118,9 +119,6 @@ The following can be set when running make: `make FOO=bar` - `DEBUG`: Disable some optimizations and enable more runtime checking - `HOST_ID_SALT`: Optional salt to use when generating host package ids - `BUILD_ID_SALT`: Optional salt to use when generating build package ids -- `FORCE_USE_SYSTEM_CLANG`: (EXPERTS ONLY) When cross-compiling for macOS, use Clang found in the - system's `$PATH` rather than the default prebuilt release of Clang - from llvm.org. Clang 8 or later is required - `LOG`: Use file-based logging for individual packages. During a package build its log file resides in the `depends` directory, and the log file is printed out automatically in case of build error. After successful build log files are moved along with package archives diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk index be04e1d8f3..2b59353e84 100644 --- a/depends/builders/darwin.mk +++ b/depends/builders/darwin.mk @@ -18,8 +18,6 @@ darwin_STRIP:=$(shell xcrun -f strip) darwin_OBJDUMP:=$(shell xcrun -f objdump) darwin_NM:=$(shell xcrun -f nm) darwin_DSYMUTIL:=$(shell xcrun -f dsymutil) -darwin_native_binutils= -darwin_native_toolchain= x86_64_darwin_CFLAGS += -arch x86_64 x86_64_darwin_CXXFLAGS += -arch x86_64 diff --git a/depends/funcs.mk b/depends/funcs.mk index 7f79478cbd..3c0dc7a7fc 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -46,7 +46,7 @@ endef define int_get_build_id $(eval $(1)_dependencies += $($(1)_$(host_arch)_$(host_os)_dependencies) $($(1)_$(host_os)_dependencies)) -$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($($(1)_type)_native_binutils) $($(1)_dependencies))) +$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($(1)_dependencies))) $(foreach dep,$($(1)_all_dependencies),$(eval $(1)_build_id_deps+=$(dep)-$($(dep)_version)-$($(dep)_recipe_hash))) $(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id)) $(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))) @@ -297,6 +297,3 @@ $(foreach package,$(all_packages),$(eval $(call int_config_attach_build_config,$ #create build targets $(foreach package,$(all_packages),$(eval $(call int_add_cmds,$(package)))) - -#special exception: if a toolchain package exists, all non-native packages depend on it -$(foreach package,$(packages),$(eval $($(package)_extracted): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) $($($(host_arch)_$(host_os)_native_binutils)_cached) )) diff --git a/depends/gen_id b/depends/gen_id index 8518b4e674..e2e2273b2d 100755 --- a/depends/gen_id +++ b/depends/gen_id @@ -53,7 +53,6 @@ echo "BEGIN AR" bash -c "${AR} --version" env | grep '^AR_' - echo "ZERO_AR_DATE=${ZERO_AR_DATE}" echo "END AR" echo "BEGIN NM" diff --git a/depends/hosts/android.mk b/depends/hosts/android.mk deleted file mode 100644 index a1c8c56dba..0000000000 --- a/depends/hosts/android.mk +++ /dev/null @@ -1,15 +0,0 @@ -ifeq ($(HOST),armv7a-linux-android) -android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)eabi$(ANDROID_API_LEVEL)-clang++ -android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)eabi$(ANDROID_API_LEVEL)-clang -else -android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang++ -android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang -endif - -android_CFLAGS=-std=$(C_STANDARD) -android_CXXFLAGS=-std=$(CXX_STANDARD) - -android_AR=$(ANDROID_TOOLCHAIN_BIN)/llvm-ar -android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/llvm-ranlib - -android_cmake_system=Android diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 2c4ef0644d..564381d1e9 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -2,31 +2,10 @@ OSX_MIN_VERSION=11.0 OSX_SDK_VERSION=14.0 XCODE_VERSION=15.0 XCODE_BUILD_ID=15A240d -LD64_VERSION=711 +LLD_VERSION=711 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers -darwin_native_binutils=native_cctools - -ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) -# FORCE_USE_SYSTEM_CLANG is empty, so we use our depends-managed, pinned clang -# from llvm.org - -# Clang is a dependency of native_cctools when FORCE_USE_SYSTEM_CLANG is empty -darwin_native_toolchain=native_cctools - -clang_prog=$(build_prefix)/bin/clang -clangxx_prog=$(clang_prog)++ -llvm_config_prog=$(build_prefix)/bin/llvm-config - -darwin_OBJDUMP=$(build_prefix)/bin/$(host)-objdump - -else -# FORCE_USE_SYSTEM_CLANG is non-empty, so we use the clang from the user's -# system - -darwin_native_toolchain= - # We can't just use $(shell command -v clang) because GNU Make handles builtins # in a special way and doesn't know that `command` is a POSIX-standard builtin # prior to 1af314465e5dfe3e8baa839a32a72e83c04f26ef, first released in v4.2.90. @@ -36,22 +15,13 @@ darwin_native_toolchain= # Source: https://lists.gnu.org/archive/html/bug-make/2017-11/msg00017.html clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang") clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++") -llvm_config_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-config") - -llvm_lib_dir=$(shell $(llvm_config_prog) --libdir) +darwin_AR=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-ar") +darwin_DSYMUTIL=$(shell $(SHELL) $(.SHELLFLAGS) "command -v dsymutil") +darwin_NM=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-nm") darwin_OBJDUMP=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-objdump") -endif - -cctools_TOOLS=AR RANLIB STRIP NM DSYMUTIL - -# Make-only lowercase function -lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1)))))))))))))))))))))))))) - -# For well-known tools provided by cctools, make sure that their well-known -# variable is set to the full path of the tool, just like how AC_PATH_{TOO,PROG} -# would. -$(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$(host)-$(call lc,$(TOOL)))) +darwin_RANLIB=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-ranlib") +darwin_STRIP=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-strip") # Flag explanations: # @@ -60,11 +30,6 @@ $(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$( # Ensures that modern linker features are enabled. See here for more # details: https://github.com/bitcoin/bitcoin/pull/19407. # -# -B$(build_prefix)/bin -# -# Explicitly point to our binaries (e.g. cctools) so that they are -# ensured to be found and preferred over other possibilities. -# # -isysroot$(OSX_SDK) -nostdlibinc # # Disable default include paths built into the compiler as well as @@ -79,12 +44,16 @@ $(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$( # # Indicate to the linker the platform, the oldest supported version, # and the SDK used. +# +# -no_adhoc_codesign +# +# Disable adhoc codesigning (for now) when using LLVM tooling, to avoid +# non-determinism issues with the Identifier field. darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ -u LIBRARY_PATH \ $(clang_prog) --target=$(host) \ - -B$(build_prefix)/bin \ -isysroot$(OSX_SDK) -nostdlibinc \ -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks @@ -92,18 +61,18 @@ darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ -u LIBRARY_PATH \ $(clangxx_prog) --target=$(host) \ - -B$(build_prefix)/bin \ -isysroot$(OSX_SDK) -nostdlibinc \ -iwithsysroot/usr/include/c++/v1 \ -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks -darwin_CFLAGS=-pipe -std=$(C_STANDARD) -mmacosx-version-min=$(OSX_MIN_VERSION) -darwin_CXXFLAGS=-pipe -std=$(CXX_STANDARD) -mmacosx-version-min=$(OSX_MIN_VERSION) +darwin_CFLAGS=-pipe -std=$(C_STANDARD) -mmacos-version-min=$(OSX_MIN_VERSION) +darwin_CXXFLAGS=-pipe -std=$(CXX_STANDARD) -mmacos-version-min=$(OSX_MIN_VERSION) darwin_LDFLAGS=-Wl,-platform_version,macos,$(OSX_MIN_VERSION),$(OSX_SDK_VERSION) ifneq ($(build_os),darwin) -darwin_CFLAGS += -mlinker-version=$(LD64_VERSION) -darwin_CXXFLAGS += -mlinker-version=$(LD64_VERSION) +darwin_CFLAGS += -mlinker-version=$(LLD_VERSION) +darwin_CXXFLAGS += -mlinker-version=$(LLD_VERSION) +darwin_LDFLAGS += -Wl,-no_adhoc_codesign -fuse-ld=lld endif darwin_release_CFLAGS=-O2 diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index 7f0389b30d..938e9971ba 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -1,6 +1,6 @@ package=boost $(package)_version=1.81.0 -$(package)_download_path=https://boostorg.jfrog.io/artifactory/main/release/$($(package)_version)/source/ +$(package)_download_path=https://archives.boost.io/release/$($(package)_version)/source/ $(package)_file_name=boost_$(subst .,_,$($(package)_version)).tar.gz $(package)_sha256_hash=205666dea9f6a7cfed87c7a6dfbeb52a2c1b9de55712c9c1a87735d7181452b6 diff --git a/depends/packages/miniupnpc.mk b/depends/packages/miniupnpc.mk index 077e3bb1ee..341031b5f8 100644 --- a/depends/packages/miniupnpc.mk +++ b/depends/packages/miniupnpc.mk @@ -1,6 +1,6 @@ package=miniupnpc $(package)_version=2.2.7 -$(package)_download_path=https://miniupnp.tuxfamily.org/files/ +$(package)_download_path=http://miniupnp.free.fr/files/ $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=b0c3a27056840fd0ec9328a5a9bac3dc5e0ec6d2e8733349cf577b0aa1e70ac1 $(package)_patches=dont_leak_info.patch cmake_get_src_addr.patch fix_windows_snprintf.patch diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk deleted file mode 100644 index 3148e51048..0000000000 --- a/depends/packages/native_cctools.mk +++ /dev/null @@ -1,39 +0,0 @@ -package=native_cctools -$(package)_version=c74fafe86076713cb8e6f937af43b6df6da1f42d -$(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive -$(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=e2c1588d505a69c32e079f4e616e0f117d5478429040e394f624f43f2796e6bc -$(package)_build_subdir=cctools -$(package)_dependencies=native_libtapi - -define $(package)_set_vars - $(package)_config_opts=--target=$(host) --enable-lto-support - $(package)_config_opts+=--with-llvm-config=$(llvm_config_prog) - $(package)_ldflags+=-Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib - $(package)_cc=$(clang_prog) - $(package)_cxx=$(clangxx_prog) -endef - -ifneq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) -define $(package)_preprocess_cmds - mkdir -p $($(package)_staging_prefix_dir)/lib && \ - cp $(llvm_lib_dir)/libLTO.so $($(package)_staging_prefix_dir)/lib/ -endef -else -endif - -define $(package)_config_cmds - $($(package)_autoconf) -endef - -define $(package)_build_cmds - $(MAKE) -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install -endef - -define $(package)_postprocess_cmds - rm -rf share -endef diff --git a/depends/packages/native_libtapi.mk b/depends/packages/native_libtapi.mk deleted file mode 100644 index fb5ab0b4dc..0000000000 --- a/depends/packages/native_libtapi.mk +++ /dev/null @@ -1,22 +0,0 @@ -package=native_libtapi -$(package)_version=eb33a59f2e30ff9724dc1ea8bee8b5229b0557c9 -$(package)_download_path=https://github.com/tpoechtrager/apple-libtapi/archive -$(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=d4d46c64622f13d6938cecf989046d9561011bb59e8ee835f8f39825d67f578f -$(package)_patches=disable_zlib.patch - -ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) -$(package)_dependencies=native_llvm -endif - -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/disable_zlib.patch -endef - -define $(package)_build_cmds - CC=$(clang_prog) CXX=$(clangxx_prog) INSTALLPREFIX=$($(package)_staging_prefix_dir) ./build.sh -endef - -define $(package)_stage_cmds - ./install.sh -endef diff --git a/depends/packages/native_llvm.mk b/depends/packages/native_llvm.mk deleted file mode 100644 index 1953c91bf4..0000000000 --- a/depends/packages/native_llvm.mk +++ /dev/null @@ -1,26 +0,0 @@ -package=native_llvm -$(package)_version=17.0.6 -$(package)_major_version=$(firstword $(subst ., ,$($(package)_version))) -$(package)_download_path=https://github.com/llvm/llvm-project/releases/download/llvmorg-$($(package)_version) -ifneq (,$(findstring aarch64,$(BUILD))) -$(package)_file_name=clang+llvm-$($(package)_version)-aarch64-linux-gnu.tar.xz -$(package)_sha256_hash=6dd62762285326f223f40b8e4f2864b5c372de3f7de0731cb7cd55ca5287b75a -else -$(package)_file_name=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-22.04.tar.xz -$(package)_sha256_hash=884ee67d647d77e58740c1e645649e29ae9e8a6fe87c1376be0f3a30f3cc9ab3 -endif - -define $(package)_stage_cmds - mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_major_version)/include && \ - mkdir -p $($(package)_staging_prefix_dir)/bin && \ - mkdir -p $($(package)_staging_prefix_dir)/include/llvm-c && \ - cp bin/clang $($(package)_staging_prefix_dir)/bin/ && \ - cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ && \ - cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \ - cp bin/llvm-config $($(package)_staging_prefix_dir)/bin/ && \ - cp bin/llvm-objdump $($(package)_staging_prefix_dir)/bin/$(host)-objdump && \ - cp include/llvm-c/ExternC.h $($(package)_staging_prefix_dir)/include/llvm-c && \ - cp include/llvm-c/lto.h $($(package)_staging_prefix_dir)/include/llvm-c && \ - cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ - cp -r lib/clang/$($(package)_major_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_major_version)/include/ -endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index fb52fd4499..01ed0d7a92 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -5,12 +5,10 @@ boost_packages = boost libevent_packages = libevent qrencode_linux_packages = qrencode -qrencode_android_packages = qrencode qrencode_darwin_packages = qrencode qrencode_mingw32_packages = qrencode qt_linux_packages:=qt expat libxcb xcb_proto libXau xproto freetype fontconfig libxkbcommon libxcb_util libxcb_util_render libxcb_util_keysyms libxcb_util_image libxcb_util_wm -qt_android_packages=qt qt_darwin_packages=qt qt_mingw32_packages=qt @@ -26,14 +24,3 @@ multiprocess_packages = libmultiprocess capnp multiprocess_native_packages = native_libmultiprocess native_capnp usdt_linux_packages=systemtap - -darwin_native_packages = - -ifneq ($(build_os),darwin) -darwin_native_packages += native_cctools native_libtapi - -ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) -darwin_native_packages+= native_llvm -endif - -endif diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index d057b2d410..d35139dd2d 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -1,9 +1,9 @@ package=qt -$(package)_version=5.15.13 +$(package)_version=5.15.14 $(package)_download_path=https://download.qt.io/official_releases/qt/5.15/$($(package)_version)/submodules $(package)_suffix=everywhere-opensource-src-$($(package)_version).tar.xz $(package)_file_name=qtbase-$($(package)_suffix) -$(package)_sha256_hash=4cca51dcc1f22ceeee6b3e33cd1c3a60b14e85e24644dca3af89a2c2989ab809 +$(package)_sha256_hash=500d3b390048e9538c28b5f523dfea6936f9c2e10d24ab46580ff57d430b98be $(package)_linux_dependencies=freetype fontconfig libxcb libxkbcommon libxcb_util libxcb_util_render libxcb_util_keysyms libxcb_util_image libxcb_util_wm $(package)_qt_libs=corelib network widgets gui plugins testlib $(package)_linguist_tools = lrelease lupdate lconvert @@ -12,23 +12,24 @@ $(package)_patches += qttools_src.pro $(package)_patches += mac-qmake.conf $(package)_patches += fix_qt_pkgconfig.patch $(package)_patches += no-xlib.patch -$(package)_patches += fix_android_jni_static.patch $(package)_patches += dont_hardcode_pwd.patch $(package)_patches += qtbase-moc-ignore-gcc-macro.patch +$(package)_patches += no_warnings_for_symbols.patch $(package)_patches += rcc_hardcode_timestamp.patch $(package)_patches += duplicate_lcqpafonts.patch $(package)_patches += guix_cross_lib_path.patch $(package)_patches += fix-macos-linker.patch $(package)_patches += memory_resource.patch +$(package)_patches += clang_18_libpng.patch $(package)_patches += utc_from_string_no_optimize.patch $(package)_patches += windows_lto.patch $(package)_patches += zlib-timebits64.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) -$(package)_qttranslations_sha256_hash=24d4c58bc2a40c0f44f59ee64af4192c7d0038c1e45af61646cfc5b65058f271 +$(package)_qttranslations_sha256_hash=5b94d1a11b566908622fcca2f8b799744d2f8a68da20be4caa5953ed63b10489 $(package)_qttools_file_name=qttools-$($(package)_suffix) -$(package)_qttools_sha256_hash=57c9794c572c4e02871f2e7581525752b0cf85ea16cfab23a4ac9ba7b39a5d34 +$(package)_qttools_sha256_hash=12061a85baf5f4de8fbc795e1d3872b706f340211b9e70962caeffc6f5e89563 $(package)_extra_sources = $($(package)_qttranslations_file_name) $(package)_extra_sources += $($(package)_qttools_file_name) @@ -138,7 +139,7 @@ ifneq ($(build_os),darwin) $(package)_config_opts_darwin += -xplatform macx-clang-linux $(package)_config_opts_darwin += -device-option MAC_SDK_PATH=$(OSX_SDK) $(package)_config_opts_darwin += -device-option MAC_SDK_VERSION=$(OSX_SDK_VERSION) -$(package)_config_opts_darwin += -device-option CROSS_COMPILE="$(host)-" +$(package)_config_opts_darwin += -device-option CROSS_COMPILE="llvm-" $(package)_config_opts_darwin += -device-option MAC_TARGET=$(host) $(package)_config_opts_darwin += -device-option XCODE_VERSION=$(XCODE_VERSION) endif @@ -185,24 +186,6 @@ $(package)_config_opts_mingw32 += -pch ifneq ($(LTO),) $(package)_config_opts_mingw32 += -ltcg endif - -$(package)_config_opts_android = -xplatform android-clang -$(package)_config_opts_android += -android-sdk $(ANDROID_SDK) -$(package)_config_opts_android += -android-ndk $(ANDROID_NDK) -$(package)_config_opts_android += -android-ndk-platform android-$(ANDROID_API_LEVEL) -$(package)_config_opts_android += -egl -$(package)_config_opts_android += -no-dbus -$(package)_config_opts_android += -opengl es2 -$(package)_config_opts_android += -qt-freetype -$(package)_config_opts_android += -no-fontconfig -$(package)_config_opts_android += -L $(host_prefix)/lib -$(package)_config_opts_android += -I $(host_prefix)/include -$(package)_config_opts_android += -pch -$(package)_config_opts_android += -no-feature-vulkan - -$(package)_config_opts_aarch64_android += -android-arch arm64-v8a -$(package)_config_opts_armv7a_android += -android-arch armeabi-v7a -$(package)_config_opts_x86_64_android += -android-arch x86_64 endef define $(package)_fetch_cmds @@ -243,10 +226,11 @@ define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/fix-macos-linker.patch && \ patch -p1 -i $($(package)_patch_dir)/dont_hardcode_pwd.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_qt_pkgconfig.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_android_jni_static.patch && \ patch -p1 -i $($(package)_patch_dir)/no-xlib.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ patch -p1 -i $($(package)_patch_dir)/memory_resource.patch && \ + patch -p1 -i $($(package)_patch_dir)/no_warnings_for_symbols.patch && \ + patch -p1 -i $($(package)_patch_dir)/clang_18_libpng.patch && \ patch -p1 -i $($(package)_patch_dir)/rcc_hardcode_timestamp.patch && \ patch -p1 -i $($(package)_patch_dir)/duplicate_lcqpafonts.patch && \ patch -p1 -i $($(package)_patch_dir)/utc_from_string_no_optimize.patch && \ diff --git a/depends/patches/native_libtapi/disable_zlib.patch b/depends/patches/native_libtapi/disable_zlib.patch deleted file mode 100644 index 6c7691214a..0000000000 --- a/depends/patches/native_libtapi/disable_zlib.patch +++ /dev/null @@ -1,17 +0,0 @@ -build: disable zlib - -This isn't needed, and causes issues when clang-tblgen -is built, but trys to reach for a system libz.so. - -diff --git a/build.sh b/build.sh -index e25d2f732..ec8422621 100755 ---- a/build.sh -+++ b/build.sh -@@ -66,6 +66,7 @@ cmake ../src/llvm \ - -DCMAKE_INSTALL_PREFIX=$INSTALLPREFIX \ - -DTAPI_REPOSITORY_STRING=$TAPI_VERSION \ - -DTAPI_FULL_VERSION=$TAPI_VERSION \ -+ -DLLVM_ENABLE_ZLIB=OFF \ - $CMAKE_EXTRA_ARGS - - echo "" diff --git a/depends/patches/qt/clang_18_libpng.patch b/depends/patches/qt/clang_18_libpng.patch new file mode 100644 index 0000000000..e807905b32 --- /dev/null +++ b/depends/patches/qt/clang_18_libpng.patch @@ -0,0 +1,40 @@ +fix Qt macOS build with Clang 18 + + See: + https://github.com/pnggroup/libpng/commit/893b8113f04d408cc6177c6de19c9889a48faa24. + + In a similar manner as zlib (madler/zlib#895), + libpng contains a header configuration that's no longer valid and + hasn't been exercised for the macOS target. + + - The target OS conditional macros are misused. Specifically + `TARGET_OS_MAC` covers all Apple targets, including iOS, and it + should not be checked with `#if defined` as they would always be + defined (to either 1 or 0) on Apple platforms. + - `#include <fp.h>` no longer works for the macOS target and results + in a compilation failure. macOS ships all required functions in + `math.h`, and clients should use `math.h` instead. + +--- a/qtbase/src/3rdparty/libpng/pngpriv.h ++++ b/qtbase/src/3rdparty/libpng/pngpriv.h +@@ -514,18 +514,8 @@ + */ + # include <float.h> + +-# if (defined(__MWERKS__) && defined(macintosh)) || defined(applec) || \ +- defined(THINK_C) || defined(__SC__) || defined(TARGET_OS_MAC) +- /* We need to check that <math.h> hasn't already been included earlier +- * as it seems it doesn't agree with <fp.h>, yet we should really use +- * <fp.h> if possible. +- */ +-# if !defined(__MATH_H__) && !defined(__MATH_H) && !defined(__cmath__) +-# include <fp.h> +-# endif +-# else +-# include <math.h> +-# endif ++# include <math.h> ++ + # if defined(_AMIGA) && defined(__SASC) && defined(_M68881) + /* Amiga SAS/C: We must include builtin FPU functions when compiling using + * MATH=68881 diff --git a/depends/patches/qt/fix_android_jni_static.patch b/depends/patches/qt/fix_android_jni_static.patch deleted file mode 100644 index 79824f244a..0000000000 --- a/depends/patches/qt/fix_android_jni_static.patch +++ /dev/null @@ -1,17 +0,0 @@ ---- old/qtbase/src/plugins/platforms/android/androidjnimain.cpp -+++ new/qtbase/src/plugins/platforms/android/androidjnimain.cpp -@@ -979,6 +979,14 @@ Q_DECL_EXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void */*reserved*/) - __android_log_print(ANDROID_LOG_FATAL, "Qt", "registerNatives failed"); - return -1; - } -+ -+ const jint ret = QT_PREPEND_NAMESPACE(QtAndroidPrivate::initJNI(vm, env)); -+ if (ret != 0) -+ { -+ __android_log_print(ANDROID_LOG_FATAL, "Qt", "initJNI failed"); -+ return ret; -+ } -+ - QWindowSystemInterfacePrivate::TabletEvent::setPlatformSynthesizesMouse(false); - - m_javaVM = vm; diff --git a/depends/patches/qt/mac-qmake.conf b/depends/patches/qt/mac-qmake.conf index cb94bf07b4..a29db20004 100644 --- a/depends/patches/qt/mac-qmake.conf +++ b/depends/patches/qt/mac-qmake.conf @@ -13,6 +13,7 @@ QMAKE_MAC_SDK.macosx.Path = $${MAC_SDK_PATH} QMAKE_MAC_SDK.macosx.platform_name = macosx QMAKE_MAC_SDK.macosx.SDKVersion = $${MAC_SDK_VERSION} QMAKE_MAC_SDK.macosx.PlatformPath = /phony +QMAKE_CXXFLAGS += -fuse-ld=lld !host_build: QMAKE_CFLAGS += -target $${MAC_TARGET} !host_build: QMAKE_OBJECTIVE_CFLAGS += $$QMAKE_CFLAGS !host_build: QMAKE_CXXFLAGS += -target $${MAC_TARGET} diff --git a/depends/patches/qt/no_warnings_for_symbols.patch b/depends/patches/qt/no_warnings_for_symbols.patch new file mode 100644 index 0000000000..11cdc599ed --- /dev/null +++ b/depends/patches/qt/no_warnings_for_symbols.patch @@ -0,0 +1,11 @@ +--- a/qtbase/mkspecs/features/mac/no_warn_empty_obj_files.prf ++++ b/qtbase/mkspecs/features/mac/no_warn_empty_obj_files.prf +@@ -1,7 +1,7 @@ + # Prevent warnings about object files without any symbols. This is a common + # thing in Qt as we tend to build files unconditionally, and then use ifdefs + # to compile out parts that are not relevant. +-QMAKE_RANLIB += -no_warning_for_no_symbols ++# QMAKE_RANLIB += -no_warning_for_no_symbols + + # We have to tell 'ar' to not run ranlib by itself + QMAKE_AR += -S diff --git a/doc/JSON-RPC-interface.md b/doc/JSON-RPC-interface.md index 7640102172..10d8ee52eb 100644 --- a/doc/JSON-RPC-interface.md +++ b/doc/JSON-RPC-interface.md @@ -33,10 +33,10 @@ requests when multiple wallets are in use. ```sh # Get block count from the / endpoint when rpcuser=alice and rpcport=38332 -$ curl --user alice --data-binary '{"jsonrpc": "1.0", "id": "0", "method": "getblockcount", "params": []}' -H 'content-type: text/plain;' localhost:38332/ +$ curl --user alice --data-binary '{"jsonrpc": "2.0", "id": "0", "method": "getblockcount", "params": []}' -H 'content-type: application/json' localhost:38332/ # Get balance from the /wallet/walletname endpoint when rpcuser=alice, rpcport=38332 and rpcwallet=desc-wallet -$ curl --user alice --data-binary '{"jsonrpc": "1.0", "id": "0", "method": "getbalance", "params": []}' -H 'content-type: text/plain;' localhost:38332/wallet/desc-wallet +$ curl --user alice --data-binary '{"jsonrpc": "2.0", "id": "0", "method": "getbalance", "params": []}' -H 'content-type: application/json' localhost:38332/wallet/desc-wallet ``` @@ -80,7 +80,7 @@ The server recognizes [JSON-RPC v2.0](https://www.jsonrpc.org/specification) req and responds accordingly. A 2.0 request is identified by the presence of `"jsonrpc": "2.0"` in the request body. If that key + value is not present in a request, the legacy JSON-RPC v1.1 protocol is followed instead, which was the only available -protocol in previous releases. +protocol in v27.0 and prior releases. || 1.1 | 2.0 | |-|-|-| @@ -88,7 +88,7 @@ protocol in previous releases. | Response marker | (none) | `"jsonrpc": "2.0"` | | `"error"` and `"result"` fields in response | both present | only one is present | | HTTP codes in response | `200` unless there is any kind of RPC error (invalid parameters, method not found, etc) | Always `200` unless there is an actual HTTP server error (request parsing error, endpoint not found, etc) | -| Notifications: requests that get no reply | (not supported) | Supported for requests that exclude the "id" field | +| Notifications: requests that get no reply | (not supported) | Supported for requests that exclude the "id" field. Returns HTTP status `204` "No Content" | ## Security diff --git a/doc/README.md b/doc/README.md index 7b6dacaf4f..74a85b04e6 100644 --- a/doc/README.md +++ b/doc/README.md @@ -45,7 +45,6 @@ The following are developer notes on how to build Bitcoin Core on your native pl - [FreeBSD Build Notes](build-freebsd.md) - [OpenBSD Build Notes](build-openbsd.md) - [NetBSD Build Notes](build-netbsd.md) -- [Android Build Notes](build-android.md) Development --------------------- diff --git a/doc/bips.md b/doc/bips.md index 8309ee7e92..19a8091f55 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -69,3 +69,4 @@ BIPs that are implemented by Bitcoin Core: [`385`](https://github.com/bitcoin/bips/blob/master/bip-0385.mediawiki): Output Script Descriptors, and most of Script Expressions are implemented as of **v0.17.0** ([PR 13697](https://github.com/bitcoin/bitcoin/pull/13697)). * [`BIP 386`](https://github.com/bitcoin/bips/blob/master/bip-0386.mediawiki): tr() Output Script Descriptors are implemented as of **v22.0** ([PR 22051](https://github.com/bitcoin/bitcoin/pull/22051)). +* [`BIP 431`](https://github.com/bitcoin/bips/blob/master/bip-0431.mediawiki): transactions with nVersion=3 are standard and treated as Topologically Restricted Until Confirmation as of **v28.0** ([PR 29496](https://github.com/bitcoin/bitcoin/pull/29496)). diff --git a/doc/bitcoin-conf.md b/doc/bitcoin-conf.md index 1ebfb4c1de..76711d0e7d 100644 --- a/doc/bitcoin-conf.md +++ b/doc/bitcoin-conf.md @@ -59,7 +59,7 @@ The `includeconf=<file>` option in the `bitcoin.conf` file can be used to includ Operating System | Data Directory | Example Path -- | -- | -- -Windows | `%APPDATA%\Bitcoin\` | `C:\Users\username\AppData\Roaming\Bitcoin\bitcoin.conf` +Windows | `%LOCALAPPDATA%\Bitcoin\` | `C:\Users\username\AppData\Local\Bitcoin\bitcoin.conf` Linux | `$HOME/.bitcoin/` | `/home/username/.bitcoin/bitcoin.conf` macOS | `$HOME/Library/Application Support/Bitcoin/` | `/Users/username/Library/Application Support/Bitcoin/bitcoin.conf` diff --git a/doc/build-android.md b/doc/build-android.md deleted file mode 100644 index 2f2e01c441..0000000000 --- a/doc/build-android.md +++ /dev/null @@ -1,25 +0,0 @@ -ANDROID BUILD NOTES -====================== - -This guide describes how to build and package the `bitcoin-qt` GUI for Android on Linux and macOS. - - -## Dependencies - -Before proceeding with an Android build one needs to get the [Android SDK](https://developer.android.com/studio) and use the "SDK Manager" tool to download the NDK and one or more "Platform packages" (these are Android versions and have a corresponding API level). - -The minimum supported Android NDK version is [r23](https://github.com/android/ndk/wiki/Changelog-r23). - -In order to build `ANDROID_API_LEVEL` (API level corresponding to the Android version targeted, e.g. Android 9.0 Pie is 28 and its "Platform package" needs to be available) and `ANDROID_TOOLCHAIN_BIN` (path to toolchain binaries depending on the platform the build is being performed on) need to be set. - -API levels from 24 to 29 have been tested to work. - -If the build includes Qt, environment variables `ANDROID_SDK` and `ANDROID_NDK` need to be set as well but can otherwise be omitted. -This is an example command for a default build with no disabled dependencies: - - ANDROID_SDK=/home/user/Android/Sdk ANDROID_NDK=/home/user/Android/Sdk/ndk-bundle make HOST=aarch64-linux-android ANDROID_API_LEVEL=28 ANDROID_TOOLCHAIN_BIN=/home/user/Android/Sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/bin - - -## Building and packaging - -After the depends are built configure with one of the resulting prefixes and run `make && make apk` in `src/qt`.
\ No newline at end of file diff --git a/doc/dependencies.md b/doc/dependencies.md index 2b9d9128d3..fc574b6164 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -30,7 +30,7 @@ You can find installation instructions in the `build-*.md` file for your platfor | [Fontconfig](../depends/packages/fontconfig.mk) | [link](https://www.freedesktop.org/wiki/Software/fontconfig/) | [2.12.6](https://github.com/bitcoin/bitcoin/pull/23495) | 2.6 | Yes | | [FreeType](../depends/packages/freetype.mk) | [link](https://freetype.org) | [2.11.0](https://github.com/bitcoin/bitcoin/commit/01544dd78ccc0b0474571da854e27adef97137fb) | 2.3.0 | Yes | | [qrencode](../depends/packages/qrencode.mk) | [link](https://fukuchi.org/works/qrencode/) | [4.1.1](https://github.com/bitcoin/bitcoin/pull/27312) | | No | -| [Qt](../depends/packages/qt.mk) | [link](https://download.qt.io/official_releases/qt/) | [5.15.13](https://github.com/bitcoin/bitcoin/pull/29732) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | +| [Qt](../depends/packages/qt.mk) | [link](https://download.qt.io/official_releases/qt/) | [5.15.14](https://github.com/bitcoin/bitcoin/pull/30198) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | ### Networking | Dependency | Releases | Version used | Minimum required | Runtime | diff --git a/doc/design/libraries.md b/doc/design/libraries.md index 251c52199d..aa8034ab37 100644 --- a/doc/design/libraries.md +++ b/doc/design/libraries.md @@ -5,6 +5,7 @@ | *libbitcoin_cli* | RPC client functionality used by *bitcoin-cli* executable | | *libbitcoin_common* | Home for common functionality shared by different executables and libraries. Similar to *libbitcoin_util*, but higher-level (see [Dependencies](#dependencies)). | | *libbitcoin_consensus* | Stable, backwards-compatible consensus functionality used by *libbitcoin_node* and *libbitcoin_wallet*. | +| *libbitcoin_crypto* | Hardware-optimized functions for data encryption, hashing, message authentication, and key derivation. | | *libbitcoin_kernel* | Consensus engine and support library used for validation by *libbitcoin_node*. | | *libbitcoinqt* | GUI functionality used by *bitcoin-qt* and *bitcoin-gui* executables. | | *libbitcoin_ipc* | IPC functionality used by *bitcoin-node*, *bitcoin-wallet*, *bitcoin-gui* executables to communicate when [`--enable-multiprocess`](multiprocess.md) is used. | @@ -53,13 +54,18 @@ bitcoin-wallet[bitcoin-wallet]-->libbitcoin_wallet_tool; libbitcoin_cli-->libbitcoin_util; libbitcoin_cli-->libbitcoin_common; +libbitcoin_consensus-->libbitcoin_crypto; + libbitcoin_common-->libbitcoin_consensus; +libbitcoin_common-->libbitcoin_crypto; libbitcoin_common-->libbitcoin_util; libbitcoin_kernel-->libbitcoin_consensus; +libbitcoin_kernel-->libbitcoin_crypto; libbitcoin_kernel-->libbitcoin_util; libbitcoin_node-->libbitcoin_consensus; +libbitcoin_node-->libbitcoin_crypto; libbitcoin_node-->libbitcoin_kernel; libbitcoin_node-->libbitcoin_common; libbitcoin_node-->libbitcoin_util; @@ -67,7 +73,10 @@ libbitcoin_node-->libbitcoin_util; libbitcoinqt-->libbitcoin_common; libbitcoinqt-->libbitcoin_util; +libbitcoin_util-->libbitcoin_crypto; + libbitcoin_wallet-->libbitcoin_common; +libbitcoin_wallet-->libbitcoin_crypto; libbitcoin_wallet-->libbitcoin_util; libbitcoin_wallet_tool-->libbitcoin_wallet; @@ -78,22 +87,23 @@ class bitcoin-qt,bitcoind,bitcoin-cli,bitcoin-wallet bold ``` </td></tr><tr><td> -**Dependency graph**. Arrows show linker symbol dependencies. *Consensus* lib depends on nothing. *Util* lib is depended on by everything. *Kernel* lib depends only on consensus and util. +**Dependency graph**. Arrows show linker symbol dependencies. *Crypto* lib depends on nothing. *Util* lib is depended on by everything. *Kernel* lib depends only on consensus, crypto, and util. </td></tr></table> - The graph shows what _linker symbols_ (functions and variables) from each library other libraries can call and reference directly, but it is not a call graph. For example, there is no arrow connecting *libbitcoin_wallet* and *libbitcoin_node* libraries, because these libraries are intended to be modular and not depend on each other's internal implementation details. But wallet code is still able to call node code indirectly through the `interfaces::Chain` abstract class in [`interfaces/chain.h`](../../src/interfaces/chain.h) and node code calls wallet code through the `interfaces::ChainClient` and `interfaces::Chain::Notifications` abstract classes in the same file. In general, defining abstract classes in [`src/interfaces/`](../../src/interfaces/) can be a convenient way of avoiding unwanted direct dependencies or circular dependencies between libraries. -- *libbitcoin_consensus* should be a standalone dependency that any library can depend on, and it should not depend on any other libraries itself. +- *libbitcoin_crypto* should be a standalone dependency that any library can depend on, and it should not depend on any other libraries itself. -- *libbitcoin_util* should also be a standalone dependency that any library can depend on, and it should not depend on other internal libraries. +- *libbitcoin_consensus* should only depend on *libbitcoin_crypto*, and all other libraries besides *libbitcoin_crypto* should be allowed to depend on it. -- *libbitcoin_common* should serve a similar function as *libbitcoin_util* and be a place for miscellaneous code used by various daemon, GUI, and CLI applications and libraries to live. It should not depend on anything other than *libbitcoin_util* and *libbitcoin_consensus*. The boundary between _util_ and _common_ is a little fuzzy but historically _util_ has been used for more generic, lower-level things like parsing hex, and _common_ has been used for bitcoin-specific, higher-level things like parsing base58. The difference between util and common is mostly important because *libbitcoin_kernel* is not supposed to depend on *libbitcoin_common*, only *libbitcoin_util*. In general, if it is ever unclear whether it is better to add code to *util* or *common*, it is probably better to add it to *common* unless it is very generically useful or useful particularly to include in the kernel. +- *libbitcoin_util* should be a standalone dependency that any library can depend on, and it should not depend on other libraries except *libbitcoin_crypto*. It provides basic utilities that fill in gaps in the C++ standard library and provide lightweight abstractions over platform-specific features. Since the util library is distributed with the kernel and is usable by kernel applications, it shouldn't contain functions that external code shouldn't call, like higher level code targetted at the node or wallet. (*libbitcoin_common* is a better place for higher level code, or code that is meant to be used by internal applications only.) +- *libbitcoin_common* is a home for miscellaneous shared code used by different Bitcoin Core applications. It should not depend on anything other than *libbitcoin_util*, *libbitcoin_consensus*, and *libbitcoin_crypto*. -- *libbitcoin_kernel* should only depend on *libbitcoin_util* and *libbitcoin_consensus*. +- *libbitcoin_kernel* should only depend on *libbitcoin_util*, *libbitcoin_consensus*, and *libbitcoin_crypto*. -- The only thing that should depend on *libbitcoin_kernel* internally should be *libbitcoin_node*. GUI and wallet libraries *libbitcoinqt* and *libbitcoin_wallet* in particular should not depend on *libbitcoin_kernel* and the unneeded functionality it would pull in, like block validation. To the extent that GUI and wallet code need scripting and signing functionality, they should be get able it from *libbitcoin_consensus*, *libbitcoin_common*, and *libbitcoin_util*, instead of *libbitcoin_kernel*. +- The only thing that should depend on *libbitcoin_kernel* internally should be *libbitcoin_node*. GUI and wallet libraries *libbitcoinqt* and *libbitcoin_wallet* in particular should not depend on *libbitcoin_kernel* and the unneeded functionality it would pull in, like block validation. To the extent that GUI and wallet code need scripting and signing functionality, they should be get able it from *libbitcoin_consensus*, *libbitcoin_common*, *libbitcoin_crypto*, and *libbitcoin_util*, instead of *libbitcoin_kernel*. - GUI, node, and wallet code internal implementations should all be independent of each other, and the *libbitcoinqt*, *libbitcoin_node*, *libbitcoin_wallet* libraries should never reference each other's symbols. They should only call each other through [`src/interfaces/`](../../src/interfaces/) abstract interfaces. diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 3458a71467..d9d5b392c5 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -13,7 +13,6 @@ Developer Notes - [Development tips and tricks](#development-tips-and-tricks) - [Compiling for debugging](#compiling-for-debugging) - [Show sources in debugging](#show-sources-in-debugging) - - [Compiling for gprof profiling](#compiling-for-gprof-profiling) - [`debug.log`](#debuglog) - [Signet, testnet, and regtest modes](#signet-testnet-and-regtest-modes) - [DEBUG_LOCKORDER](#debug_lockorder) @@ -386,10 +385,6 @@ ln -s /path/to/project/root/src src 3. Use `debugedit` to modify debug information in the binary. -### Compiling for gprof profiling - -Run configure with the `--enable-gprof` option, then make. - ### `debug.log` If the code is behaving strangely, take a look in the `debug.log` file in the data directory; @@ -497,6 +492,10 @@ make cov # unit and functional tests. ``` +Additional LCOV options can be specified using `LCOV_OPTS`, but may be dependant +on the version of LCOV. For example, when using LCOV `2.x`, branch coverage can be +enabled by setting `LCOV_OPTS="--rc branch_coverage=1"`, when configuring. + ### Performance profiling with perf Profiling is a good way to get a precise idea of where time is being spent in @@ -1458,8 +1457,9 @@ independent (node, wallet, GUI), are defined in there are [`interfaces::Chain`](../src/interfaces/chain.h), used by wallet to access the node's latest chain state, [`interfaces::Node`](../src/interfaces/node.h), used by the GUI to control the -node, and [`interfaces::Wallet`](../src/interfaces/wallet.h), used by the GUI -to control an individual wallet. There are also more specialized interface +node, [`interfaces::Wallet`](../src/interfaces/wallet.h), used by the GUI +to control an individual wallet and [`interfaces::Mining`](../src/interfaces/mining.h), +used by RPC to generate block templates. There are also more specialized interface types like [`interfaces::Handler`](../src/interfaces/handler.h) [`interfaces::ChainClient`](../src/interfaces/chain.h) passed to and from various interface methods. diff --git a/doc/files.md b/doc/files.md index f88d3f91a1..03e52f02c9 100644 --- a/doc/files.md +++ b/doc/files.md @@ -28,7 +28,7 @@ Platform | Data directory path ---------|-------------------- Linux | `$HOME/.bitcoin/` macOS | `$HOME/Library/Application Support/Bitcoin/` -Windows | `%APPDATA%\Bitcoin\` <sup>[\[1\]](#note1)</sup> +Windows | `%LOCALAPPDATA%\Bitcoin\` <sup>[\[1\]](#note1)</sup> 2. A custom data directory path can be specified with the `-datadir` option. diff --git a/doc/init.md b/doc/init.md index 7f79027718..64ab971557 100644 --- a/doc/init.md +++ b/doc/init.md @@ -43,8 +43,8 @@ This allows for running bitcoind without having to do any manual configuration. `conf`, `pid`, and `wallet` accept relative paths which are interpreted as relative to the data directory. `wallet` *only* supports relative paths. -For an example configuration file that describes the configuration settings, -see `share/examples/bitcoin.conf`. +To generate an example configuration file that describes the configuration settings, +see [contrib/devtools/README.md](../contrib/devtools/README.md#gen-bitcoin-confsh). Paths --------------------------------- diff --git a/doc/managing-wallets.md b/doc/managing-wallets.md index b99d88877b..1cfd2e5123 100644 --- a/doc/managing-wallets.md +++ b/doc/managing-wallets.md @@ -20,7 +20,7 @@ By default, wallets are created in the `wallets` folder of the data directory, w | Operating System | Default wallet directory | | -----------------|:------------------------------------------------------------| | Linux | `/home/<user>/.bitcoin/wallets` | -| Windows | `C:\Users\<user>\AppData\Roaming\Bitcoin\wallets` | +| Windows | `C:\Users\<user>\AppData\Local\Bitcoin\wallets` | | macOS | `/Users/<user>/Library/Application Support/Bitcoin/wallets` | ### 1.2 Encrypting the Wallet diff --git a/doc/policy/mempool-replacements.md b/doc/policy/mempool-replacements.md index 3fd7ff2ad2..d5642eaccc 100644 --- a/doc/policy/mempool-replacements.md +++ b/doc/policy/mempool-replacements.md @@ -12,7 +12,7 @@ other consensus and policy rules, each of the following conditions are met: 1. The directly conflicting transactions all signal replaceability explicitly. A transaction is signaling BIP125 replaceability if any of its inputs have an nSequence number less than (0xffffffff - 1). - A transaction also signals replaceability if its nVersion field is set to 3. + A transaction also signals replaceability if its version field is set to 3. *Rationale*: See [BIP125 explanation](https://github.com/bitcoin/bips/blob/master/bip-0125.mediawiki#motivation). diff --git a/doc/policy/packages.md b/doc/policy/packages.md index dba270e494..a220bdd17f 100644 --- a/doc/policy/packages.md +++ b/doc/policy/packages.md @@ -36,10 +36,29 @@ The following rules are enforced for all packages: * Packages cannot have conflicting transactions, i.e. no two transactions in a package can spend the same inputs. Packages cannot have duplicate transactions. (#20833) -* No transaction in a package can conflict with a mempool transaction. Replace By Fee is - currently disabled for packages. (#20833) +* Only limited package replacements are currently considered. (#28984) - - Package RBF may be enabled in the future. + - All direct conflicts must signal replacement (or have `-mempoolfullrbf=1` set). + + - Packages are 1-parent-1-child, with no in-mempool ancestors of the package. + + - All conflicting clusters(connected components of mempool transactions) must be clusters of up to size 2. + + - No more than MAX_REPLACEMENT_CANDIDATES transactions can be replaced, analogous to + regular [replacement rule](./mempool-replacements.md) 5). + + - Replacements must pay more total total fees at the incremental relay fee (analogous to + regular [replacement rules](./mempool-replacements.md) 3 and 4). + + - Parent feerate must be lower than package feerate. + + - Must improve [feerate diagram](https://delvingbitcoin.org/t/mempool-incentive-compatibility/553). (#29242) + + - *Rationale*: Basic support for package RBF can be used by wallets + by making chains of no longer than two, then directly conflicting + those chains when needed. Combined with V3 transactions this can + result in more robust fee bumping. More general package RBF may be + enabled in the future. * When packages are evaluated against ancestor/descendant limits, the union of all transactions' descendants and ancestors is considered. (#21800) @@ -48,8 +67,13 @@ The following rules are enforced for all packages: heavily connected, i.e. some transaction in the package is the ancestor or descendant of all the other transactions. -The following rules are only enforced for packages to be submitted to the mempool (not enforced for -test accepts): +* [CPFP Carve Out](./mempool-limits.md#CPFP-Carve-Out) is disabled in packaged contexts. (#21800) + + - *Rationale*: This carve out cannot be accurately applied when there are multiple transactions' + ancestors and descendants being considered at the same time. + +The following rules are only enforced for packages to be submitted to the mempool (not +enforced for test accepts): * Packages must be child-with-unconfirmed-parents packages. This also means packages must contain at least 2 transactions. (#22674) diff --git a/doc/release-28984.md b/doc/release-28984.md new file mode 100644 index 0000000000..3da64f6578 --- /dev/null +++ b/doc/release-28984.md @@ -0,0 +1,6 @@ +P2P and network changes +----------------------- + +- Limited package RBF is now enabled, where the proposed conflicting package would result in + a connected component, aka cluster, of size 2 in the mempool. All clusters being conflicted + against must be of size 2 or lower. diff --git a/doc/release-notes-27101.md b/doc/release-notes-27101.md index 8775b59c00..7ce1e9a8c1 100644 --- a/doc/release-notes-27101.md +++ b/doc/release-notes-27101.md @@ -2,8 +2,5 @@ JSON-RPC -------- The JSON-RPC server now recognizes JSON-RPC 2.0 requests and responds with -strict adherence to the specification (https://www.jsonrpc.org/specification): - -- Returning HTTP "204 No Content" responses to JSON-RPC 2.0 notifications instead of full responses. -- Returning HTTP "200 OK" responses in all other cases, rather than 404 responses for unknown methods, 500 responses for invalid parameters, etc. -- Returning either "result" fields or "error" fields in JSON-RPC responses, rather than returning both fields with one field set to null. +strict adherence to the [specification](https://www.jsonrpc.org/specification). +See [JSON-RPC-interface.md](/doc/JSON-RPC-interface.md#json-rpc-11-vs-20) for details.
\ No newline at end of file diff --git a/doc/release-notes-29091-29165.md b/doc/release-notes-29091-29165.md new file mode 100644 index 0000000000..9c9f8e4e50 --- /dev/null +++ b/doc/release-notes-29091-29165.md @@ -0,0 +1,5 @@ +Build +----- + +GCC 11.1 or later, or Clang 15+ or later, +are now required to compile Bitcoin Core. diff --git a/doc/release-notes-29612.md b/doc/release-notes-29612.md new file mode 100644 index 0000000000..31af3cab09 --- /dev/null +++ b/doc/release-notes-29612.md @@ -0,0 +1,8 @@ +RPC +--- + +- The `dumptxoutset` RPC now returns the UTXO set dump in a new and + improved format. At the same time the `loadtxoutset` RPC now + expects this new format in dumps it tries to load. Dumps with the + old format are no longer supported and need to be recreated using + the new format in order to be usable. diff --git a/doc/release-notes-30058.md b/doc/release-notes-30058.md new file mode 100644 index 0000000000..47e7ae704e --- /dev/null +++ b/doc/release-notes-30058.md @@ -0,0 +1,7 @@ +- When running with -alertnotify, an alert can now be raised multiple +times instead of just once. Previously, it was only raised when unknown +new consensus rules were activated, whereas the scope has now been +increased to include all kernel warnings. Specifically, alerts will now +also be raised when an invalid chain with a large amount of work has +been detected. Additional warnings may be added in the future. +(#30058) diff --git a/doc/release-notes-30192.md b/doc/release-notes-30192.md new file mode 100644 index 0000000000..2a6c17d455 --- /dev/null +++ b/doc/release-notes-30192.md @@ -0,0 +1,6 @@ +Build +----- + +`--enable-lcov-branch-coverage` has been removed, given +incompatibilities between lcov version 1 & 2. `LCOV_OPTS` +should be used to set any options instead. diff --git a/doc/release-notes/release-notes-27.1.md b/doc/release-notes/release-notes-27.1.md new file mode 100644 index 0000000000..b19d70da33 --- /dev/null +++ b/doc/release-notes/release-notes-27.1.md @@ -0,0 +1,114 @@ +27.1 Release Notes +===================== + +Bitcoin Core version 27.1 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-27.1/> + +This release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux Kernel 3.17+, macOS 11.0+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +### Miniscript + +- #29853 sign: don't assume we are parsing a sane TapMiniscript + +### RPC + +- #29869 rpc, bugfix: Enforce maximum value for setmocktime +- #29870 rpc: Reword SighashFromStr error message +- #30094 rpc: move UniValue in blockToJSON + +### Index + +- #29776 Fix #29767, set m_synced = true after Commit() + +### Gui + +- #gui812 Fix create unsigned transaction fee bump +- #gui813 Don't permit port in proxy IP option + +### Test + +- #29892 test: Fix failing univalue float test + +### P2P + +- #30085 p2p: detect addnode cjdns peers in GetAddedNodeInfo() + +### Build + +- #29747 depends: fix mingw-w64 Qt DEBUG=1 build +- #29859 build: Fix false positive CHECK_ATOMIC test +- #29985 depends: Fix build of Qt for 32-bit platforms with recent glibc +- #30097 crypto: disable asan for sha256_sse4 with clang and -O0 +- #30151 depends: Fetch miniupnpc sources from an alternative website +- #30216 build: Fix building fuzz binary on on SunOS / illumos +- #30217 depends: Update Boost download link + +### Doc + +- #29934 doc: add LLVM instruction for macOS < 13 + +### CI + +- #29856 ci: Bump s390x to ubuntu:24.04 + +### Misc + +- #29691 Change Luke Dashjr seed to dashjr-list-of-p2p-nodes.us +- #30149 contrib: Renew Windows code signing certificate + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Antoine Poinsot +- Ava Chow +- Cory Fields +- dergoegge +- fanquake +- furszy +- Hennadii Stepanov +- Jon Atack +- laanwj +- Luke Dashjr +- MarcoFalke +- nanlour +- Sjors Provoost +- willcl-ark + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-27064.md b/doc/release-notes/release-notes-27064.md new file mode 100644 index 0000000000..be3ecee1b8 --- /dev/null +++ b/doc/release-notes/release-notes-27064.md @@ -0,0 +1,7 @@ +Files +----- + +The default data directory on Windows has been moved from `C:\Users\Username\AppData\Roaming\Bitcoin` +to `C:\Users\Username\AppData\Local\Bitcoin`. Bitcoin Core will check the existence +of the old directory first and continue to use that directory for backwards +compatibility if it is present.
\ No newline at end of file diff --git a/src/.clang-tidy b/src/.clang-tidy index a00400f083..61adce1d50 100644 --- a/src/.clang-tidy +++ b/src/.clang-tidy @@ -2,6 +2,7 @@ Checks: ' -*, bitcoin-*, bugprone-argument-comment, +bugprone-move-forwarding-reference, bugprone-string-constructor, bugprone-use-after-move, bugprone-lambda-function-name, diff --git a/src/Makefile.am b/src/Makefile.am index ad37928b4d..0c47a737d0 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -8,8 +8,8 @@ print-%: FORCE DIST_SUBDIRS = secp256k1 -AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) $(CORE_LDFLAGS) -AM_CXXFLAGS = $(CORE_CXXFLAGS) $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) +AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(SANITIZER_LDFLAGS) $(CORE_LDFLAGS) +AM_CXXFLAGS = $(CORE_CXXFLAGS) $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(SANITIZER_CXXFLAGS) AM_OBJCXXFLAGS = $(AM_CXXFLAGS) AM_CPPFLAGS = $(DEBUG_CPPFLAGS) $(HARDENED_CPPFLAGS) $(CORE_CPPFLAGS) AM_LIBTOOLFLAGS = --preserve-dup-deps @@ -137,13 +137,16 @@ BITCOIN_CORE_H = \ common/bloom.h \ common/init.h \ common/run_command.h \ + common/types.h \ common/url.h \ compat/assumptions.h \ compat/byteswap.h \ compat/compat.h \ compat/cpuid.h \ compat/endian.h \ + common/messages.h \ common/settings.h \ + common/signmessage.h \ common/system.h \ compressor.h \ consensus/consensus.h \ @@ -174,6 +177,7 @@ BITCOIN_CORE_H = \ interfaces/handler.h \ interfaces/init.h \ interfaces/ipc.h \ + interfaces/mining.h \ interfaces/node.h \ interfaces/wallet.h \ kernel/blockmanager_opts.h \ @@ -193,6 +197,7 @@ BITCOIN_CORE_H = \ kernel/messagestartchars.h \ kernel/notifications_interface.h \ kernel/validation_cache_sizes.h \ + kernel/warning.h \ key.h \ key_io.h \ logging.h \ @@ -233,8 +238,10 @@ BITCOIN_CORE_H = \ node/timeoffsets.h \ node/transaction.h \ node/txreconciliation.h \ + node/types.h \ node/utxo_snapshot.h \ node/validation_cache_args.h \ + node/warnings.h \ noui.h \ outputtype.h \ policy/v3_policy.h \ @@ -267,6 +274,7 @@ BITCOIN_CORE_H = \ script/descriptor.h \ script/keyorigin.h \ script/miniscript.h \ + script/parsing.h \ script/sigcache.h \ script/sign.h \ script/signingprovider.h \ @@ -292,15 +300,14 @@ BITCOIN_CORE_H = \ util/batchpriority.h \ util/bip32.h \ util/bitdeque.h \ + util/bitset.h \ util/bytevectorhash.h \ util/chaintype.h \ util/check.h \ util/epochguard.h \ - util/error.h \ util/exception.h \ util/fastrange.h \ util/feefrac.h \ - util/fees.h \ util/fs.h \ util/fs_helpers.h \ util/golombrice.h \ @@ -308,7 +315,6 @@ BITCOIN_CORE_H = \ util/hasher.h \ util/insert.h \ util/macros.h \ - util/message.h \ util/moneystr.h \ util/overflow.h \ util/overloaded.h \ @@ -318,7 +324,7 @@ BITCOIN_CORE_H = \ util/serfloat.h \ util/signalinterrupt.h \ util/sock.h \ - util/spanparsing.h \ + util/strencodings.h \ util/string.h \ util/subprocess.h \ util/syserror.h \ @@ -333,6 +339,7 @@ BITCOIN_CORE_H = \ util/translation.h \ util/types.h \ util/ui_change_type.h \ + util/vecdeque.h \ util/vector.h \ validation.h \ validationinterface.h \ @@ -363,7 +370,6 @@ BITCOIN_CORE_H = \ wallet/wallettool.h \ wallet/walletutil.h \ walletinitinterface.h \ - warnings.h \ zmq/zmqabstractnotifier.h \ zmq/zmqnotificationinterface.h \ zmq/zmqpublishnotifier.h \ @@ -440,6 +446,7 @@ libbitcoin_node_a_SOURCES = \ node/txreconciliation.cpp \ node/utxo_snapshot.cpp \ node/validation_cache_args.cpp \ + node/warnings.cpp \ noui.cpp \ policy/v3_policy.cpp \ policy/fees.cpp \ @@ -567,6 +574,8 @@ crypto_libbitcoin_crypto_base_la_SOURCES = \ crypto/chacha20poly1305.h \ crypto/chacha20poly1305.cpp \ crypto/common.h \ + crypto/hex_base.cpp \ + crypto/hex_base.h \ crypto/hkdf_sha256_32.cpp \ crypto/hkdf_sha256_32.h \ crypto/hmac_sha256.cpp \ @@ -589,7 +598,8 @@ crypto_libbitcoin_crypto_base_la_SOURCES = \ crypto/sha512.cpp \ crypto/sha512.h \ crypto/siphash.cpp \ - crypto/siphash.h + crypto/siphash.h \ + support/cleanse.cpp # See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and # CXXFLAGS above @@ -659,9 +669,7 @@ libbitcoin_consensus_a_SOURCES = \ span.h \ tinyformat.h \ uint256.cpp \ - uint256.h \ - util/strencodings.cpp \ - util/strencodings.h + uint256.h # # common # @@ -671,6 +679,7 @@ libbitcoin_common_a_SOURCES = \ addresstype.cpp \ base58.cpp \ bech32.cpp \ + chainparamsbase.cpp \ chainparams.cpp \ coins.cpp \ common/args.cpp \ @@ -678,8 +687,10 @@ libbitcoin_common_a_SOURCES = \ common/config.cpp \ common/init.cpp \ common/interfaces.cpp \ + common/messages.cpp \ common/run_command.cpp \ common/settings.cpp \ + common/signmessage.cpp \ common/system.cpp \ common/url.cpp \ compressor.cpp \ @@ -709,10 +720,10 @@ libbitcoin_common_a_SOURCES = \ scheduler.cpp \ script/descriptor.cpp \ script/miniscript.cpp \ + script/parsing.cpp \ script/sign.cpp \ script/signingprovider.cpp \ script/solver.cpp \ - warnings.cpp \ $(BITCOIN_CORE_H) # @@ -721,13 +732,11 @@ libbitcoin_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_util_a_SOURCES = \ support/lockedpool.cpp \ - chainparamsbase.cpp \ clientversion.cpp \ logging.cpp \ random.cpp \ randomenv.cpp \ streams.cpp \ - support/cleanse.cpp \ sync.cpp \ util/asmap.cpp \ util/batchpriority.cpp \ @@ -735,16 +744,13 @@ libbitcoin_util_a_SOURCES = \ util/bytevectorhash.cpp \ util/chaintype.cpp \ util/check.cpp \ - util/error.cpp \ util/exception.cpp \ util/feefrac.cpp \ - util/fees.cpp \ util/fs.cpp \ util/fs_helpers.cpp \ util/hasher.cpp \ util/sock.cpp \ util/syserror.cpp \ - util/message.cpp \ util/moneystr.cpp \ util/rbf.cpp \ util/readwritefile.cpp \ @@ -753,7 +759,6 @@ libbitcoin_util_a_SOURCES = \ util/threadinterrupt.cpp \ util/threadnames.cpp \ util/serfloat.cpp \ - util/spanparsing.cpp \ util/strencodings.cpp \ util/string.cpp \ util/time.cpp \ @@ -970,7 +975,6 @@ libbitcoinkernel_la_SOURCES = \ script/solver.cpp \ signet.cpp \ streams.cpp \ - support/cleanse.cpp \ support/lockedpool.cpp \ sync.cpp \ txdb.cpp \ @@ -994,8 +998,7 @@ libbitcoinkernel_la_SOURCES = \ util/tokenpipe.cpp \ validation.cpp \ validationinterface.cpp \ - versionbits.cpp \ - warnings.cpp + versionbits.cpp # Required for obj/build.h to be generated first. # More details: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html @@ -1086,7 +1089,7 @@ libbitcoin_ipc_a_SOURCES = \ ipc/process.cpp \ ipc/process.h \ ipc/protocol.h -libbitcoin_ipc_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +libbitcoin_ipc_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libbitcoin_ipc_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) $(LIBMULTIPROCESS_CFLAGS) include $(MPGEN_PREFIX)/include/mpgen.mk diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 7ba0111fa6..2ba72c9e76 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -23,6 +23,7 @@ bench_bench_bitcoin_SOURCES = \ bench/ccoins_caching.cpp \ bench/chacha20.cpp \ bench/checkblock.cpp \ + bench/checkblockindex.cpp \ bench/checkqueue.cpp \ bench/crypto_hash.cpp \ bench/data.cpp \ diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index c235c3c4da..1ac6b74688 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -392,20 +392,6 @@ bitcoin_qt_clean: FORCE bitcoin_qt : qt/bitcoin-qt$(EXEEXT) -APK_LIB_DIR = qt/android/libs/$(ANDROID_ARCH) -QT_BASE_VERSION = $(lastword $(shell $(MOC) --version)) -QT_BASE_PATH = $(shell find ../depends/sources/ -maxdepth 1 -type f -regex ".*qtbase.*$(QT_BASE_VERSION)\.tar.xz") -QT_BASE_TLD = $(shell tar tf $(QT_BASE_PATH) --exclude='*/*') - -bitcoin_qt_apk: FORCE - mkdir -p $(APK_LIB_DIR) - cp $(dir $(lastword $(CC)))../sysroot/usr/lib/$(host_alias)/libc++_shared.so $(APK_LIB_DIR) - tar xf $(QT_BASE_PATH) -C qt/android/src/ $(QT_BASE_TLD)src/android/jar/src --strip-components=5 - tar xf $(QT_BASE_PATH) -C qt/android/src/ $(QT_BASE_TLD)src/android/java/src --strip-components=5 - cp qt/bitcoin-qt $(APK_LIB_DIR)/libbitcoin-qt_$(ANDROID_ARCH).so - cd qt/android && gradle wrapper --gradle-version=6.6.1 - cd qt/android && ./gradlew build - ui_%.h: %.ui @test -f $(UIC) || (echo "uic $(UIC) not found, but is required for generating ui headers"; exit 1) @$(MKDIR_P) $(@D) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 62a82189c1..633d0776f5 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -118,6 +118,7 @@ BITCOIN_TESTS =\ test/net_peer_eviction_tests.cpp \ test/net_tests.cpp \ test/netbase_tests.cpp \ + test/node_warnings_tests.cpp \ test/orphanage_tests.cpp \ test/peerman_tests.cpp \ test/pmt_tests.cpp \ @@ -203,6 +204,7 @@ endif FUZZ_WALLET_SRC = \ wallet/test/fuzz/coincontrol.cpp \ wallet/test/fuzz/coinselection.cpp \ + wallet/test/fuzz/crypter.cpp \ wallet/test/fuzz/fees.cpp \ wallet/test/fuzz/parse_iso8601.cpp \ wallet/test/fuzz/wallet_bdb_parser.cpp @@ -277,7 +279,7 @@ FUZZ_SUITE_LD_COMMON += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif if ENABLE_FUZZ_BINARY -test_fuzz_fuzz_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) +test_fuzz_fuzz_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) $(EVENT_CFLAGS) test_fuzz_fuzz_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_fuzz_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_fuzz_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) @@ -293,6 +295,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/bech32.cpp \ test/fuzz/bip324.cpp \ test/fuzz/bitdeque.cpp \ + test/fuzz/bitset.cpp \ test/fuzz/block.cpp \ test/fuzz/block_header.cpp \ test/fuzz/blockfilter.cpp \ @@ -326,6 +329,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/headerssync.cpp \ test/fuzz/hex.cpp \ test/fuzz/http_request.cpp \ + test/fuzz/i2p.cpp \ test/fuzz/integer.cpp \ test/fuzz/key.cpp \ test/fuzz/key_io.cpp \ @@ -372,6 +376,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/script_format.cpp \ test/fuzz/script_interpreter.cpp \ test/fuzz/script_ops.cpp \ + test/fuzz/script_parsing.cpp \ test/fuzz/script_sigcache.cpp \ test/fuzz/script_sign.cpp \ test/fuzz/scriptnum_ops.cpp \ @@ -381,7 +386,6 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/signet.cpp \ test/fuzz/socks5.cpp \ test/fuzz/span.cpp \ - test/fuzz/spanparsing.cpp \ test/fuzz/string.cpp \ test/fuzz/strprintf.cpp \ test/fuzz/system.cpp \ @@ -396,6 +400,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/utxo_snapshot.cpp \ test/fuzz/utxo_total_supply.cpp \ test/fuzz/validation_load_mempool.cpp \ + test/fuzz/vecdeque.cpp \ test/fuzz/versionbits.cpp endif # ENABLE_FUZZ_BINARY diff --git a/src/banman.h b/src/banman.h index c6df7ec3c3..57ba2ac23c 100644 --- a/src/banman.h +++ b/src/banman.h @@ -34,7 +34,7 @@ class CSubNet; // disk on shutdown and reloaded on startup. Banning can be used to // prevent connections with spy nodes or other griefers. // -// 2. Discouragement. If a peer misbehaves enough (see Misbehaving() in +// 2. Discouragement. If a peer misbehaves (see Misbehaving() in // net_processing.cpp), we'll mark that address as discouraged. We still allow // incoming connections from them, but they're preferred for eviction when // we receive new incoming connections. We never make outgoing connections to diff --git a/src/base58.cpp b/src/base58.cpp index cf5d62f164..f9165ed55f 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -14,6 +14,8 @@ #include <limits> +using util::ContainsNoNUL; + /** All alphanumeric characters except for "0", "I", "O", and "l" */ static const char* pszBase58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; static const int8_t mapBase58[256] = { diff --git a/src/bech32.cpp b/src/bech32.cpp index ba3c419d8b..d8d31a415c 100644 --- a/src/bech32.cpp +++ b/src/bech32.cpp @@ -19,6 +19,9 @@ namespace typedef std::vector<uint8_t> data; +/** The Bech32 and Bech32m checksum size */ +constexpr size_t CHECKSUM_SIZE = 6; + /** The Bech32 and Bech32m character set for encoding. */ const char* CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l"; @@ -308,18 +311,18 @@ bool CheckCharacters(const std::string& str, std::vector<int>& errors) return errors.empty(); } -/** Expand a HRP for use in checksum computation. */ -data ExpandHRP(const std::string& hrp) +std::vector<unsigned char> PreparePolynomialCoefficients(const std::string& hrp, const data& values) { data ret; - ret.reserve(hrp.size() + 90); - ret.resize(hrp.size() * 2 + 1); - for (size_t i = 0; i < hrp.size(); ++i) { - unsigned char c = hrp[i]; - ret[i] = c >> 5; - ret[i + hrp.size() + 1] = c & 0x1f; - } - ret[hrp.size()] = 0; + ret.reserve(hrp.size() + 1 + hrp.size() + values.size() + CHECKSUM_SIZE); + + /** Expand a HRP for use in checksum computation. */ + for (size_t i = 0; i < hrp.size(); ++i) ret.push_back(hrp[i] >> 5); + ret.push_back(0); + for (size_t i = 0; i < hrp.size(); ++i) ret.push_back(hrp[i] & 0x1f); + + ret.insert(ret.end(), values.begin(), values.end()); + return ret; } @@ -331,7 +334,8 @@ Encoding VerifyChecksum(const std::string& hrp, const data& values) // list of values would result in a new valid list. For that reason, Bech32 requires the // resulting checksum to be 1 instead. In Bech32m, this constant was amended. See // https://gist.github.com/sipa/14c248c288c3880a3b191f978a34508e for details. - const uint32_t check = PolyMod(Cat(ExpandHRP(hrp), values)); + auto enc = PreparePolynomialCoefficients(hrp, values); + const uint32_t check = PolyMod(enc); if (check == EncodingConstant(Encoding::BECH32)) return Encoding::BECH32; if (check == EncodingConstant(Encoding::BECH32M)) return Encoding::BECH32M; return Encoding::INVALID; @@ -340,11 +344,11 @@ Encoding VerifyChecksum(const std::string& hrp, const data& values) /** Create a checksum. */ data CreateChecksum(Encoding encoding, const std::string& hrp, const data& values) { - data enc = Cat(ExpandHRP(hrp), values); - enc.resize(enc.size() + 6); // Append 6 zeroes + auto enc = PreparePolynomialCoefficients(hrp, values); + enc.insert(enc.end(), CHECKSUM_SIZE, 0x00); uint32_t mod = PolyMod(enc) ^ EncodingConstant(encoding); // Determine what to XOR into those 6 zeroes. - data ret(6); - for (size_t i = 0; i < 6; ++i) { + data ret(CHECKSUM_SIZE); + for (size_t i = 0; i < CHECKSUM_SIZE; ++i) { // Convert the 5-bit groups in mod to checksum values. ret[i] = (mod >> (5 * (5 - i))) & 31; } @@ -359,22 +363,23 @@ std::string Encode(Encoding encoding, const std::string& hrp, const data& values // to return a lowercase Bech32/Bech32m string, but if given an uppercase HRP, the // result will always be invalid. for (const char& c : hrp) assert(c < 'A' || c > 'Z'); - data checksum = CreateChecksum(encoding, hrp, values); - data combined = Cat(values, checksum); - std::string ret = hrp + '1'; - ret.reserve(ret.size() + combined.size()); - for (const auto c : combined) { - ret += CHARSET[c]; - } + + std::string ret; + ret.reserve(hrp.size() + 1 + values.size() + CHECKSUM_SIZE); + ret += hrp; + ret += '1'; + for (const uint8_t& i : values) ret += CHARSET[i]; + for (const uint8_t& i : CreateChecksum(encoding, hrp, values)) ret += CHARSET[i]; return ret; } /** Decode a Bech32 or Bech32m string. */ -DecodeResult Decode(const std::string& str) { +DecodeResult Decode(const std::string& str, CharLimit limit) { std::vector<int> errors; if (!CheckCharacters(str, errors)) return {}; size_t pos = str.rfind('1'); - if (str.size() > 90 || pos == str.npos || pos == 0 || pos + 7 > str.size()) { + if (str.size() > limit) return {}; + if (pos == str.npos || pos == 0 || pos + CHECKSUM_SIZE >= str.size()) { return {}; } data values(str.size() - 1 - pos); @@ -388,21 +393,22 @@ DecodeResult Decode(const std::string& str) { values[i] = rev; } std::string hrp; + hrp.reserve(pos); for (size_t i = 0; i < pos; ++i) { hrp += LowerCase(str[i]); } Encoding result = VerifyChecksum(hrp, values); if (result == Encoding::INVALID) return {}; - return {result, std::move(hrp), data(values.begin(), values.end() - 6)}; + return {result, std::move(hrp), data(values.begin(), values.end() - CHECKSUM_SIZE)}; } /** Find index of an incorrect character in a Bech32 string. */ -std::pair<std::string, std::vector<int>> LocateErrors(const std::string& str) { +std::pair<std::string, std::vector<int>> LocateErrors(const std::string& str, CharLimit limit) { std::vector<int> error_locations{}; - if (str.size() > 90) { - error_locations.resize(str.size() - 90); - std::iota(error_locations.begin(), error_locations.end(), 90); + if (str.size() > limit) { + error_locations.resize(str.size() - limit); + std::iota(error_locations.begin(), error_locations.end(), static_cast<int>(limit)); return std::make_pair("Bech32 string too long", std::move(error_locations)); } @@ -414,12 +420,13 @@ std::pair<std::string, std::vector<int>> LocateErrors(const std::string& str) { if (pos == str.npos) { return std::make_pair("Missing separator", std::vector<int>{}); } - if (pos == 0 || pos + 7 > str.size()) { + if (pos == 0 || pos + CHECKSUM_SIZE >= str.size()) { error_locations.push_back(pos); return std::make_pair("Invalid separator position", std::move(error_locations)); } std::string hrp; + hrp.reserve(pos); for (size_t i = 0; i < pos; ++i) { hrp += LowerCase(str[i]); } @@ -441,9 +448,10 @@ std::pair<std::string, std::vector<int>> LocateErrors(const std::string& str) { std::optional<Encoding> error_encoding; for (Encoding encoding : {Encoding::BECH32, Encoding::BECH32M}) { std::vector<int> possible_errors; - // Recall that (ExpandHRP(hrp) ++ values) is interpreted as a list of coefficients of a polynomial + // Recall that (expanded hrp + values) is interpreted as a list of coefficients of a polynomial // over GF(32). PolyMod computes the "remainder" of this polynomial modulo the generator G(x). - uint32_t residue = PolyMod(Cat(ExpandHRP(hrp), values)) ^ EncodingConstant(encoding); + auto enc = PreparePolynomialCoefficients(hrp, values); + uint32_t residue = PolyMod(enc) ^ EncodingConstant(encoding); // All valid codewords should be multiples of G(x), so this remainder (after XORing with the encoding // constant) should be 0 - hence 0 indicates there are no errors present. diff --git a/src/bech32.h b/src/bech32.h index 5e89e6efda..fe2a276ae0 100644 --- a/src/bech32.h +++ b/src/bech32.h @@ -28,6 +28,14 @@ enum class Encoding { BECH32M, //!< Bech32m encoding as defined in BIP350 }; +/** Character limits for Bech32(m) encoded strings. Character limits are how we provide error location guarantees. + * These values should never exceed 2^31 - 1 (max value for a 32-bit int), since there are places where we may need to + * convert the CharLimit::VALUE to an int. In practice, this should never happen since this CharLimit applies to an address encoding + * and we would never encode an address with such a massive value */ +enum CharLimit : size_t { + BECH32 = 90, //!< BIP173/350 imposed character limit for Bech32(m) encoded addresses. This guarantees finding up to 4 errors. +}; + /** Encode a Bech32 or Bech32m string. If hrp contains uppercase characters, this will cause an * assertion error. Encoding must be one of BECH32 or BECH32M. */ std::string Encode(Encoding encoding, const std::string& hrp, const std::vector<uint8_t>& values); @@ -43,10 +51,10 @@ struct DecodeResult }; /** Decode a Bech32 or Bech32m string. */ -DecodeResult Decode(const std::string& str); +DecodeResult Decode(const std::string& str, CharLimit limit = CharLimit::BECH32); /** Return the positions of errors in a Bech32 string. */ -std::pair<std::string, std::vector<int>> LocateErrors(const std::string& str); +std::pair<std::string, std::vector<int>> LocateErrors(const std::string& str, CharLimit limit = CharLimit::BECH32); } // namespace bech32 diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index a13a693ad7..733f8085ca 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -18,6 +18,7 @@ #include <vector> using namespace std::chrono_literals; +using util::Join; const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index 8c421c3fec..a1b880e40b 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -16,6 +16,8 @@ #include <sstream> #include <vector> +using util::SplitString; + static const char* DEFAULT_BENCH_FILTER = ".*"; static constexpr int64_t DEFAULT_MIN_TIME_MS{10}; /** Priority level default value, run "all" priority levels */ diff --git a/src/bench/checkblockindex.cpp b/src/bench/checkblockindex.cpp new file mode 100644 index 0000000000..e8a848dbd4 --- /dev/null +++ b/src/bench/checkblockindex.cpp @@ -0,0 +1,20 @@ +// Copyright (c) 2023-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <test/util/setup_common.h> +#include <validation.h> + +static void CheckBlockIndex(benchmark::Bench& bench) +{ + auto testing_setup{MakeNoLogFileContext<TestChain100Setup>()}; + // Mine some more blocks + testing_setup->mineBlocks(1000); + bench.run([&] { + testing_setup->m_node.chainman->CheckBlockIndex(); + }); +} + + +BENCHMARK(CheckBlockIndex, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 249b76ee85..171c61c46f 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -71,15 +71,15 @@ static void CoinSelection(benchmark::Bench& bench) /*change_output_size=*/ 34, /*change_spend_size=*/ 148, /*min_change_target=*/ CHANGE_LOWER, - /*effective_feerate=*/ CFeeRate(0), - /*long_term_feerate=*/ CFeeRate(0), - /*discard_feerate=*/ CFeeRate(0), + /*effective_feerate=*/ CFeeRate(20'000), + /*long_term_feerate=*/ CFeeRate(10'000), + /*discard_feerate=*/ CFeeRate(3000), /*tx_noinputs_size=*/ 0, /*avoid_partial=*/ false, }; auto group = wallet::GroupOutputs(wallet, available_coins, coin_selection_params, {{filter_standard}})[filter_standard]; bench.run([&] { - auto result = AttemptSelection(wallet.chain(), 1003 * COIN, group, coin_selection_params, /*allow_mixed_output_types=*/true); + auto result = AttemptSelection(wallet.chain(), 1002.99 * COIN, group, coin_selection_params, /*allow_mixed_output_types=*/true); assert(result); assert(result->GetSelectedValue() == 1003 * COIN); assert(result->GetInputSet().size() == 2); diff --git a/src/bench/wallet_create.cpp b/src/bench/wallet_create.cpp index 618d5bc80c..5c0557bf6f 100644 --- a/src/bench/wallet_create.cpp +++ b/src/bench/wallet_create.cpp @@ -34,14 +34,15 @@ static void WalletCreate(benchmark::Bench& bench, bool encrypted) bilingual_str error_string; std::vector<bilingual_str> warnings; - fs::path wallet_path = test_setup->m_path_root / strprintf("test_wallet_%d", random.rand32()).c_str(); + auto wallet_path = fs::PathToString(test_setup->m_path_root / "test_wallet"); bench.run([&] { - auto wallet = CreateWallet(context, wallet_path.utf8string(), /*load_on_start=*/std::nullopt, options, status, error_string, warnings); + auto wallet = CreateWallet(context, wallet_path, /*load_on_start=*/std::nullopt, options, status, error_string, warnings); assert(status == DatabaseStatus::SUCCESS); assert(wallet != nullptr); - // Cleanup - wallet.reset(); + // Release wallet + RemoveWallet(context, wallet, /*load_on_start=*/ std::nullopt); + UnloadWallet(std::move(wallet)); fs::remove_all(wallet_path); }); } @@ -49,14 +50,9 @@ static void WalletCreate(benchmark::Bench& bench, bool encrypted) static void WalletCreatePlain(benchmark::Bench& bench) { WalletCreate(bench, /*encrypted=*/false); } static void WalletCreateEncrypted(benchmark::Bench& bench) { WalletCreate(bench, /*encrypted=*/true); } -#ifndef _MSC_VER -// TODO: Being built with MSVC, the fs::remove_all() call in -// the WalletCreate() fails with the error "The process cannot -// access the file because it is being used by another process." #ifdef USE_SQLITE BENCHMARK(WalletCreatePlain, benchmark::PriorityLevel::LOW); BENCHMARK(WalletCreateEncrypted, benchmark::PriorityLevel::LOW); #endif -#endif } // namespace wallet diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index 4d2a6f0c2a..ecbdcd48bb 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -16,6 +16,7 @@ #include <kernel/checks.h> #include <kernel/context.h> #include <kernel/validation_cache_sizes.h> +#include <kernel/warning.h> #include <consensus/validation.h> #include <core_io.h> @@ -28,6 +29,7 @@ #include <util/fs.h> #include <util/signalinterrupt.h> #include <util/task_runner.h> +#include <util/translation.h> #include <validation.h> #include <validationinterface.h> @@ -86,9 +88,13 @@ int main(int argc, char* argv[]) { std::cout << "Progress: " << title.original << ", " << progress_percent << ", " << resume_possible << std::endl; } - void warning(const bilingual_str& warning) override + void warningSet(kernel::Warning id, const bilingual_str& message) override { - std::cout << "Warning: " << warning.original << std::endl; + std::cout << "Warning " << static_cast<int>(id) << " set: " << message.original << std::endl; + } + void warningUnset(kernel::Warning id) override + { + std::cout << "Warning " << static_cast<int>(id) << " unset" << std::endl; } void flushError(const bilingual_str& message) override { @@ -151,7 +157,7 @@ int main(int argc, char* argv[]) { LOCK(chainman.GetMutex()); std::cout - << "\t" << "Reindexing: " << std::boolalpha << chainman.m_blockman.m_reindexing.load() << std::noboolalpha << std::endl + << "\t" << "Blockfiles Indexed: " << std::boolalpha << chainman.m_blockman.m_blockfiles_indexed.load() << std::noboolalpha << std::endl << "\t" << "Snapshot Active: " << std::boolalpha << chainman.IsSnapshotActive() << std::noboolalpha << std::endl << "\t" << "Active Height: " << chainman.ActiveHeight() << std::endl << "\t" << "Active IBD: " << std::boolalpha << chainman.IsInitialBlockDownload() << std::noboolalpha << std::endl; diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index b7e4e64103..44fc273163 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -42,6 +42,9 @@ #include <event2/keyvalq_struct.h> #include <support/events.h> +using util::Join; +using util::ToString; + // The server returns time values from a mockable system clock, but it is not // trivial to get the mocked time from the server, nor is it needed for now, so // just use a plain system_clock. @@ -297,8 +300,8 @@ public: total += counts.at(i); } addresses.pushKV("total", total); - result.pushKV("addresses_known", addresses); - return JSONRPCReplyObj(std::move(result), NullUniValue, /*id=*/1, JSONRPCVersion::V1_LEGACY); + result.pushKV("addresses_known", std::move(addresses)); + return JSONRPCReplyObj(std::move(result), NullUniValue, /*id=*/1, JSONRPCVersion::V2); } }; @@ -348,7 +351,7 @@ public: connections.pushKV("in", batch[ID_NETWORKINFO]["result"]["connections_in"]); connections.pushKV("out", batch[ID_NETWORKINFO]["result"]["connections_out"]); connections.pushKV("total", batch[ID_NETWORKINFO]["result"]["connections"]); - result.pushKV("connections", connections); + result.pushKV("connections", std::move(connections)); result.pushKV("networks", batch[ID_NETWORKINFO]["result"]["networks"]); result.pushKV("difficulty", batch[ID_BLOCKCHAININFO]["result"]["difficulty"]); @@ -367,7 +370,7 @@ public: } result.pushKV("relayfee", batch[ID_NETWORKINFO]["result"]["relayfee"]); result.pushKV("warnings", batch[ID_NETWORKINFO]["result"]["warnings"]); - return JSONRPCReplyObj(std::move(result), NullUniValue, /*id=*/1, JSONRPCVersion::V1_LEGACY); + return JSONRPCReplyObj(std::move(result), NullUniValue, /*id=*/1, JSONRPCVersion::V2); } }; @@ -622,7 +625,7 @@ public: } } - return JSONRPCReplyObj(UniValue{result}, NullUniValue, /*id=*/1, JSONRPCVersion::V1_LEGACY); + return JSONRPCReplyObj(UniValue{result}, NullUniValue, /*id=*/1, JSONRPCVersion::V2); } const std::string m_help_doc{ @@ -709,7 +712,7 @@ public: UniValue result(UniValue::VOBJ); result.pushKV("address", address_str); result.pushKV("blocks", reply.get_obj()["result"]); - return JSONRPCReplyObj(std::move(result), NullUniValue, /*id=*/1, JSONRPCVersion::V1_LEGACY); + return JSONRPCReplyObj(std::move(result), NullUniValue, /*id=*/1, JSONRPCVersion::V2); } protected: std::string address_str; @@ -743,8 +746,41 @@ static UniValue CallRPC(BaseRequestHandler* rh, const std::string& strMethod, co // 2. port in -rpcconnect (ie following : in ipv4 or ]: in ipv6) // 3. default port for chain uint16_t port{BaseParams().RPCPort()}; - SplitHostPort(gArgs.GetArg("-rpcconnect", DEFAULT_RPCCONNECT), port, host); - port = static_cast<uint16_t>(gArgs.GetIntArg("-rpcport", port)); + { + uint16_t rpcconnect_port{0}; + const std::string rpcconnect_str = gArgs.GetArg("-rpcconnect", DEFAULT_RPCCONNECT); + if (!SplitHostPort(rpcconnect_str, rpcconnect_port, host)) { + // Uses argument provided as-is + // (rather than value parsed) + // to aid the user in troubleshooting + throw std::runtime_error(strprintf("Invalid port provided in -rpcconnect: %s", rpcconnect_str)); + } else { + if (rpcconnect_port != 0) { + // Use the valid port provided in rpcconnect + port = rpcconnect_port; + } // else, no port was provided in rpcconnect (continue using default one) + } + + if (std::optional<std::string> rpcport_arg = gArgs.GetArg("-rpcport")) { + // -rpcport was specified + const uint16_t rpcport_int{ToIntegral<uint16_t>(rpcport_arg.value()).value_or(0)}; + if (rpcport_int == 0) { + // Uses argument provided as-is + // (rather than value parsed) + // to aid the user in troubleshooting + throw std::runtime_error(strprintf("Invalid port provided in -rpcport: %s", rpcport_arg.value())); + } + + // Use the valid port provided + port = rpcport_int; + + // If there was a valid port provided in rpcconnect, + // rpcconnect_port is non-zero. + if (rpcconnect_port != 0) { + tfm::format(std::cerr, "Warning: Port specified in both -rpcconnect and -rpcport. Using -rpcport %u\n", port); + } + } + } // Obtain event base raii_event_base base = obtain_event_base(); @@ -940,7 +976,7 @@ static void GetWalletBalances(UniValue& result) const UniValue& balance = getbalances.find_value("result")["mine"]["trusted"]; balances.pushKV(wallet_name, balance); } - result.pushKV("balances", balances); + result.pushKV("balances", std::move(balances)); } /** diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index cfac50e090..89faa0123a 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -32,6 +32,11 @@ #include <functional> #include <memory> +using util::SplitString; +using util::ToString; +using util::TrimString; +using util::TrimStringView; + static bool fCreateBlank; static std::map<std::string,UniValue> registers; static const int CONTINUE_EXECUTION=-1; @@ -203,12 +208,12 @@ static CAmount ExtractAndValidateValue(const std::string& strValue) static void MutateTxVersion(CMutableTransaction& tx, const std::string& cmdVal) { - int64_t newVersion; - if (!ParseInt64(cmdVal, &newVersion) || newVersion < 1 || newVersion > TX_MAX_STANDARD_VERSION) { + uint32_t newVersion; + if (!ParseUInt32(cmdVal, &newVersion) || newVersion < 1 || newVersion > TX_MAX_STANDARD_VERSION) { throw std::runtime_error("Invalid TX version requested: '" + cmdVal + "'"); } - tx.nVersion = (int) newVersion; + tx.version = newVersion; } static void MutateTxLocktime(CMutableTransaction& tx, const std::string& cmdVal) diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 87af347473..b6f5c3f15d 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -24,6 +24,8 @@ #include <string> #include <tuple> +using util::Join; + const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; static void SetupWalletToolArgs(ArgsManager& argsman) diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 0b89aa42af..a09bb5c9da 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -17,6 +17,7 @@ #include <kernel/context.h> #include <node/context.h> #include <node/interface_ui.h> +#include <node/warnings.h> #include <noui.h> #include <util/check.h> #include <util/exception.h> @@ -181,6 +182,8 @@ static bool AppInit(NodeContext& node) return false; } + node.warnings = std::make_unique<node::Warnings>(); + node.kernel = std::make_unique<kernel::Context>(); node.ecc_context = std::make_unique<ECC_Context>(); if (!AppInitSanityChecks(*node.kernel)) diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index e045b88513..5e6702ccc3 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -16,6 +16,8 @@ #include <util/golombrice.h> #include <util/string.h> +using util::Join; + static const std::map<BlockFilterType, std::string> g_filter_types = { {BlockFilterType::BASIC, "basic"}, }; diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 539578085b..5d4401b719 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -21,6 +21,8 @@ #include <stdexcept> #include <vector> +using util::SplitString; + void ReadSigNetArgs(const ArgsManager& args, CChainParams::SigNetOptions& options) { if (args.IsArgSet("-signetseednode")) { diff --git a/src/clientversion.cpp b/src/clientversion.cpp index 6b9727a158..e52703c8bf 100644 --- a/src/clientversion.cpp +++ b/src/clientversion.cpp @@ -13,6 +13,8 @@ #include <string> #include <vector> +using util::Join; + /** * Name of client reported in the 'version' message. Report the same name * for both bitcoind and bitcoin-qt, to make it harder for attackers to diff --git a/src/coins.cpp b/src/coins.cpp index b62653e0de..a4e4d4ad32 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -361,7 +361,7 @@ static bool ExecuteBackedWrapper(Func func, const std::vector<std::function<void for (const auto& f : err_callbacks) { f(); } - LogPrintf("Error reading from database: %s\n", e.what()); + LogError("Error reading from database: %s\n", e.what()); // Starting the shutdown sequence and returning false to the caller would be // interpreted as 'entry not found' (as opposed to unable to read data), and // could lead to invalid interpretation. Just exit immediately, as we can't diff --git a/src/common/args.cpp b/src/common/args.cpp index c90eb0c685..caff36fdb3 100644 --- a/src/common/args.cpp +++ b/src/common/args.cpp @@ -696,12 +696,19 @@ bool HasTestOption(const ArgsManager& args, const std::string& test_option) fs::path GetDefaultDataDir() { - // Windows: C:\Users\Username\AppData\Roaming\Bitcoin + // Windows: + // old: C:\Users\Username\AppData\Roaming\Bitcoin + // new: C:\Users\Username\AppData\Local\Bitcoin // macOS: ~/Library/Application Support/Bitcoin // Unix-like: ~/.bitcoin #ifdef WIN32 // Windows - return GetSpecialFolderPath(CSIDL_APPDATA) / "Bitcoin"; + // Check for existence of datadir in old location and keep it there + fs::path legacy_path = GetSpecialFolderPath(CSIDL_APPDATA) / "Bitcoin"; + if (fs::exists(legacy_path)) return legacy_path; + + // Otherwise, fresh installs can start in the new, "proper" location + return GetSpecialFolderPath(CSIDL_LOCAL_APPDATA) / "Bitcoin"; #else fs::path pathRet; char* pszHome = getenv("HOME"); diff --git a/src/common/config.cpp b/src/common/config.cpp index 1c85273f69..98223fc3e3 100644 --- a/src/common/config.cpp +++ b/src/common/config.cpp @@ -27,6 +27,9 @@ #include <utility> #include <vector> +using util::TrimString; +using util::TrimStringView; + static bool GetConfigOptions(std::istream& stream, const std::string& filepath, std::string& error, std::vector<std::pair<std::string, std::string>>& options, std::list<SectionInfo>& sections) { std::string str, prefix; diff --git a/src/common/messages.cpp b/src/common/messages.cpp new file mode 100644 index 0000000000..9e88ca8b0f --- /dev/null +++ b/src/common/messages.cpp @@ -0,0 +1,139 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <common/messages.h> + +#include <common/types.h> +#include <policy/fees.h> +#include <node/types.h> +#include <tinyformat.h> +#include <util/strencodings.h> +#include <util/string.h> +#include <util/translation.h> + +#include <cassert> +#include <map> +#include <string> +#include <utility> +#include <vector> + +using node::TransactionError; +using util::Join; + +namespace common { +std::string StringForFeeReason(FeeReason reason) +{ + static const std::map<FeeReason, std::string> fee_reason_strings = { + {FeeReason::NONE, "None"}, + {FeeReason::HALF_ESTIMATE, "Half Target 60% Threshold"}, + {FeeReason::FULL_ESTIMATE, "Target 85% Threshold"}, + {FeeReason::DOUBLE_ESTIMATE, "Double Target 95% Threshold"}, + {FeeReason::CONSERVATIVE, "Conservative Double Target longer horizon"}, + {FeeReason::MEMPOOL_MIN, "Mempool Min Fee"}, + {FeeReason::PAYTXFEE, "PayTxFee set"}, + {FeeReason::FALLBACK, "Fallback fee"}, + {FeeReason::REQUIRED, "Minimum Required Fee"}, + }; + auto reason_string = fee_reason_strings.find(reason); + + if (reason_string == fee_reason_strings.end()) return "Unknown"; + + return reason_string->second; +} + +const std::vector<std::pair<std::string, FeeEstimateMode>>& FeeModeMap() +{ + static const std::vector<std::pair<std::string, FeeEstimateMode>> FEE_MODES = { + {"unset", FeeEstimateMode::UNSET}, + {"economical", FeeEstimateMode::ECONOMICAL}, + {"conservative", FeeEstimateMode::CONSERVATIVE}, + }; + return FEE_MODES; +} + +std::string FeeModes(const std::string& delimiter) +{ + return Join(FeeModeMap(), delimiter, [&](const std::pair<std::string, FeeEstimateMode>& i) { return i.first; }); +} + +std::string InvalidEstimateModeErrorMessage() +{ + return "Invalid estimate_mode parameter, must be one of: \"" + FeeModes("\", \"") + "\""; +} + +bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode) +{ + auto searchkey = ToUpper(mode_string); + for (const auto& pair : FeeModeMap()) { + if (ToUpper(pair.first) == searchkey) { + fee_estimate_mode = pair.second; + return true; + } + } + return false; +} + +bilingual_str PSBTErrorString(PSBTError err) +{ + switch (err) { + case PSBTError::MISSING_INPUTS: + return Untranslated("Inputs missing or spent"); + case PSBTError::SIGHASH_MISMATCH: + return Untranslated("Specified sighash value does not match value stored in PSBT"); + case PSBTError::EXTERNAL_SIGNER_NOT_FOUND: + return Untranslated("External signer not found"); + case PSBTError::EXTERNAL_SIGNER_FAILED: + return Untranslated("External signer failed to sign"); + case PSBTError::UNSUPPORTED: + return Untranslated("Signer does not support PSBT"); + // no default case, so the compiler can warn about missing cases + } + assert(false); +} + +bilingual_str TransactionErrorString(const TransactionError err) +{ + switch (err) { + case TransactionError::OK: + return Untranslated("No error"); + case TransactionError::MISSING_INPUTS: + return Untranslated("Inputs missing or spent"); + case TransactionError::ALREADY_IN_CHAIN: + return Untranslated("Transaction already in block chain"); + case TransactionError::MEMPOOL_REJECTED: + return Untranslated("Transaction rejected by mempool"); + case TransactionError::MEMPOOL_ERROR: + return Untranslated("Mempool internal error"); + case TransactionError::MAX_FEE_EXCEEDED: + return Untranslated("Fee exceeds maximum configured by user (e.g. -maxtxfee, maxfeerate)"); + case TransactionError::MAX_BURN_EXCEEDED: + return Untranslated("Unspendable output exceeds maximum configured by user (maxburnamount)"); + case TransactionError::INVALID_PACKAGE: + return Untranslated("Transaction rejected due to invalid package"); + // no default case, so the compiler can warn about missing cases + } + assert(false); +} + +bilingual_str ResolveErrMsg(const std::string& optname, const std::string& strBind) +{ + return strprintf(_("Cannot resolve -%s address: '%s'"), optname, strBind); +} + +bilingual_str InvalidPortErrMsg(const std::string& optname, const std::string& invalid_value) +{ + return strprintf(_("Invalid port specified in %s: '%s'"), optname, invalid_value); +} + +bilingual_str AmountHighWarn(const std::string& optname) +{ + return strprintf(_("%s is set very high!"), optname); +} + +bilingual_str AmountErrMsg(const std::string& optname, const std::string& strValue) +{ + return strprintf(_("Invalid amount for -%s=<amount>: '%s'"), optname, strValue); +} +} // namespace common diff --git a/src/common/messages.h b/src/common/messages.h new file mode 100644 index 0000000000..68e7bb2169 --- /dev/null +++ b/src/common/messages.h @@ -0,0 +1,38 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +//! @file common/messages.h is a home for simple string functions returning +//! descriptive messages that are used in RPC and GUI interfaces or log +//! messages, and are called in different parts of the codebase across +//! node/wallet/gui boundaries. + +#ifndef BITCOIN_COMMON_MESSAGES_H +#define BITCOIN_COMMON_MESSAGES_H + +#include <string> + +struct bilingual_str; + +enum class FeeEstimateMode; +enum class FeeReason; +namespace node { +enum class TransactionError; +} // namespace node + +namespace common { +enum class PSBTError; +bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode); +std::string StringForFeeReason(FeeReason reason); +std::string FeeModes(const std::string& delimiter); +std::string InvalidEstimateModeErrorMessage(); +bilingual_str PSBTErrorString(PSBTError error); +bilingual_str TransactionErrorString(const node::TransactionError error); +bilingual_str ResolveErrMsg(const std::string& optname, const std::string& strBind); +bilingual_str InvalidPortErrMsg(const std::string& optname, const std::string& strPort); +bilingual_str AmountHighWarn(const std::string& optname); +bilingual_str AmountErrMsg(const std::string& optname, const std::string& strValue); +} // namespace common + +#endif // BITCOIN_COMMON_MESSAGES_H diff --git a/src/util/message.cpp b/src/common/signmessage.cpp index 1afb28cc10..1612751e44 100644 --- a/src/util/message.cpp +++ b/src/common/signmessage.cpp @@ -3,12 +3,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/signmessage.h> #include <hash.h> #include <key.h> #include <key_io.h> #include <pubkey.h> #include <uint256.h> -#include <util/message.h> #include <util/strencodings.h> #include <cassert> diff --git a/src/util/message.h b/src/common/signmessage.h index d0e2422574..215b563bbb 100644 --- a/src/util/message.h +++ b/src/common/signmessage.h @@ -3,8 +3,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_UTIL_MESSAGE_H -#define BITCOIN_UTIL_MESSAGE_H +#ifndef BITCOIN_COMMON_SIGNMESSAGE_H +#define BITCOIN_COMMON_SIGNMESSAGE_H #include <uint256.h> @@ -74,4 +74,4 @@ uint256 MessageHash(const std::string& message); std::string SigningResultString(const SigningResult res); -#endif // BITCOIN_UTIL_MESSAGE_H +#endif // BITCOIN_COMMON_SIGNMESSAGE_H diff --git a/src/common/system.cpp b/src/common/system.cpp index ddd0feda3b..6d04c8a7bc 100644 --- a/src/common/system.cpp +++ b/src/common/system.cpp @@ -28,6 +28,8 @@ #include <string> #include <thread> +using util::ReplaceAll; + // Application startup time (used for uptime calculation) const int64_t nStartupTime = GetTime(); diff --git a/src/common/types.h b/src/common/types.h new file mode 100644 index 0000000000..0d9cb67ce9 --- /dev/null +++ b/src/common/types.h @@ -0,0 +1,26 @@ +// Copyright (c) 2010-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +//! @file common/types.h is a home for simple enum and struct type definitions +//! that can be used internally by functions in the libbitcoin_common library, +//! but also used externally by node, wallet, and GUI code. +//! +//! This file is intended to define only simple types that do not have external +//! dependencies. More complicated types should be defined in dedicated header +//! files. + +#ifndef BITCOIN_COMMON_TYPES_H +#define BITCOIN_COMMON_TYPES_H + +namespace common { +enum class PSBTError { + MISSING_INPUTS, + SIGHASH_MISMATCH, + EXTERNAL_SIGNER_NOT_FOUND, + EXTERNAL_SIGNER_FAILED, + UNSUPPORTED, +}; +} // namespace common + +#endif // BITCOIN_COMMON_TYPES_H diff --git a/src/compressor.h b/src/compressor.h index 0968454679..a0970c595e 100644 --- a/src/compressor.h +++ b/src/compressor.h @@ -55,8 +55,8 @@ struct ScriptCompression { /** * make this static for now (there are only 6 special scripts defined) - * this can potentially be extended together with a new nVersion for - * transactions, in which case this value becomes dependent on nVersion + * this can potentially be extended together with a new version for + * transactions, in which case this value becomes dependent on version * and nHeight of the enclosing transaction. */ static const unsigned int nSpecialScripts = 6; diff --git a/src/consensus/tx_verify.cpp b/src/consensus/tx_verify.cpp index 154146f08d..95466b759c 100644 --- a/src/consensus/tx_verify.cpp +++ b/src/consensus/tx_verify.cpp @@ -48,11 +48,7 @@ std::pair<int, int64_t> CalculateSequenceLocks(const CTransaction &tx, int flags int nMinHeight = -1; int64_t nMinTime = -1; - // tx.nVersion is signed integer so requires cast to unsigned otherwise - // we would be doing a signed comparison and half the range of nVersion - // wouldn't support BIP 68. - bool fEnforceBIP68 = static_cast<uint32_t>(tx.nVersion) >= 2 - && flags & LOCKTIME_VERIFY_SEQUENCE; + bool fEnforceBIP68 = tx.version >= 2 && flags & LOCKTIME_VERIFY_SEQUENCE; // Do not enforce sequence numbers as a relative lock time // unless we have been instructed to diff --git a/src/core_read.cpp b/src/core_read.cpp index 5956d9df5f..114f92fc45 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -16,6 +16,8 @@ #include <algorithm> #include <string> +using util::SplitString; + namespace { class OpCodeParser { diff --git a/src/core_write.cpp b/src/core_write.cpp index 63f2c36b5a..253dfde100 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -174,9 +174,7 @@ void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry entry.pushKV("txid", tx.GetHash().GetHex()); entry.pushKV("hash", tx.GetWitnessHash().GetHex()); - // Transaction version is actually unsigned in consensus checks, just signed in memory, - // so cast to unsigned before giving it to the user. - entry.pushKV("version", static_cast<int64_t>(static_cast<uint32_t>(tx.nVersion))); + entry.pushKV("version", tx.version); entry.pushKV("size", tx.GetTotalSize()); entry.pushKV("vsize", (GetTransactionWeight(tx) + WITNESS_SCALE_FACTOR - 1) / WITNESS_SCALE_FACTOR); entry.pushKV("weight", GetTransactionWeight(tx)); @@ -201,14 +199,14 @@ void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry UniValue o(UniValue::VOBJ); o.pushKV("asm", ScriptToAsmStr(txin.scriptSig, true)); o.pushKV("hex", HexStr(txin.scriptSig)); - in.pushKV("scriptSig", o); + in.pushKV("scriptSig", std::move(o)); } if (!tx.vin[i].scriptWitness.IsNull()) { UniValue txinwitness(UniValue::VARR); for (const auto& item : tx.vin[i].scriptWitness.stack) { txinwitness.push_back(HexStr(item)); } - in.pushKV("txinwitness", txinwitness); + in.pushKV("txinwitness", std::move(txinwitness)); } if (have_undo) { const Coin& prev_coin = txundo->vprevout[i]; @@ -224,14 +222,14 @@ void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry p.pushKV("generated", bool(prev_coin.fCoinBase)); p.pushKV("height", uint64_t(prev_coin.nHeight)); p.pushKV("value", ValueFromAmount(prev_txout.nValue)); - p.pushKV("scriptPubKey", o_script_pub_key); - in.pushKV("prevout", p); + p.pushKV("scriptPubKey", std::move(o_script_pub_key)); + in.pushKV("prevout", std::move(p)); } } in.pushKV("sequence", (int64_t)txin.nSequence); - vin.push_back(in); + vin.push_back(std::move(in)); } - entry.pushKV("vin", vin); + entry.pushKV("vin", std::move(vin)); UniValue vout(UniValue::VARR); for (unsigned int i = 0; i < tx.vout.size(); i++) { @@ -244,14 +242,14 @@ void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry UniValue o(UniValue::VOBJ); ScriptToUniv(txout.scriptPubKey, /*out=*/o, /*include_hex=*/true, /*include_address=*/true); - out.pushKV("scriptPubKey", o); - vout.push_back(out); + out.pushKV("scriptPubKey", std::move(o)); + vout.push_back(std::move(out)); if (have_undo) { amt_total_out += txout.nValue; } } - entry.pushKV("vout", vout); + entry.pushKV("vout", std::move(vout)); if (have_undo) { const CAmount fee = amt_total_in - amt_total_out; diff --git a/src/crypto/hex_base.cpp b/src/crypto/hex_base.cpp new file mode 100644 index 0000000000..67d691b63e --- /dev/null +++ b/src/crypto/hex_base.cpp @@ -0,0 +1,67 @@ +// Copyright (c) 2009-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <crypto/hex_base.h> + +#include <array> +#include <cstring> +#include <string> + +namespace { + +using ByteAsHex = std::array<char, 2>; + +constexpr std::array<ByteAsHex, 256> CreateByteToHexMap() +{ + constexpr char hexmap[16] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + + std::array<ByteAsHex, 256> byte_to_hex{}; + for (size_t i = 0; i < byte_to_hex.size(); ++i) { + byte_to_hex[i][0] = hexmap[i >> 4]; + byte_to_hex[i][1] = hexmap[i & 15]; + } + return byte_to_hex; +} + +} // namespace + +std::string HexStr(const Span<const uint8_t> s) +{ + std::string rv(s.size() * 2, '\0'); + static constexpr auto byte_to_hex = CreateByteToHexMap(); + static_assert(sizeof(byte_to_hex) == 512); + + char* it = rv.data(); + for (uint8_t v : s) { + std::memcpy(it, byte_to_hex[v].data(), 2); + it += 2; + } + + assert(it == rv.data() + rv.size()); + return rv; +} + +const signed char p_util_hexdigit[256] = +{ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + 0,1,2,3,4,5,6,7,8,9,-1,-1,-1,-1,-1,-1, + -1,0xa,0xb,0xc,0xd,0xe,0xf,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,0xa,0xb,0xc,0xd,0xe,0xf,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, }; + +signed char HexDigit(char c) +{ + return p_util_hexdigit[(unsigned char)c]; +} + diff --git a/src/crypto/hex_base.h b/src/crypto/hex_base.h new file mode 100644 index 0000000000..cdfea68c29 --- /dev/null +++ b/src/crypto/hex_base.h @@ -0,0 +1,23 @@ +// Copyright (c) 2009-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_HEX_BASE_H +#define BITCOIN_CRYPTO_HEX_BASE_H + +#include <span.h> + +#include <cstddef> +#include <cstdint> +#include <string> + +/** + * Convert a span of bytes to a lower-case hexadecimal string. + */ +std::string HexStr(const Span<const uint8_t> s); +inline std::string HexStr(const Span<const char> s) { return HexStr(MakeUCharSpan(s)); } +inline std::string HexStr(const Span<const std::byte> s) { return HexStr(MakeUCharSpan(s)); } + +signed char HexDigit(char c); + +#endif // BITCOIN_CRYPTO_HEX_BASE_H diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 3eb34dbe6a..128597157d 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -23,6 +23,9 @@ #include <string> #include <vector> +using util::SplitString; +using util::TrimStringView; + /** WWW-Authenticate to present with 401 Unauthorized response */ static const char* WWW_AUTH_HEADER_DATA = "Basic realm=\"jsonrpc\""; diff --git a/src/i2p.cpp b/src/i2p.cpp index 962adb124d..a907cfeacb 100644 --- a/src/i2p.cpp +++ b/src/i2p.cpp @@ -12,12 +12,12 @@ #include <netaddress.h> #include <netbase.h> #include <random.h> +#include <script/parsing.h> #include <sync.h> #include <tinyformat.h> #include <util/fs.h> #include <util/readwritefile.h> #include <util/sock.h> -#include <util/spanparsing.h> #include <util/strencodings.h> #include <util/threadinterrupt.h> @@ -26,6 +26,8 @@ #include <stdexcept> #include <string> +using util::Split; + namespace i2p { /** @@ -308,7 +310,7 @@ Session::Reply Session::SendRequestAndGetReply(const Sock& sock, reply.full = sock.RecvUntilTerminator('\n', recv_timeout, *m_interrupt, MAX_MSG_SIZE); - for (const auto& kv : spanparsing::Split(reply.full, ' ')) { + for (const auto& kv : Split(reply.full, ' ')) { const auto& pos = std::find(kv.begin(), kv.end(), '='); if (pos != kv.end()) { reply.keys.emplace(std::string{kv.begin(), pos}, std::string{pos + 1, kv.end()}); diff --git a/src/index/base.cpp b/src/index/base.cpp index e66c89f9e4..955d7b67c9 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -17,7 +17,6 @@ #include <util/thread.h> #include <util/translation.h> #include <validation.h> // For g_chainman -#include <warnings.h> #include <string> #include <utility> @@ -31,7 +30,7 @@ template <typename... Args> void BaseIndex::FatalErrorf(const char* fmt, const Args&... args) { auto message = tfm::format(fmt, args...); - node::AbortNode(m_chain->context()->shutdown, m_chain->context()->exit_status, Untranslated(message)); + node::AbortNode(m_chain->context()->shutdown, m_chain->context()->exit_status, Untranslated(message), m_chain->context()->warnings.get()); } CBlockLocator GetLocator(interfaces::Chain& chain, const uint256& block_hash) diff --git a/src/init.cpp b/src/init.cpp index 0aac2ac65f..c6ef62372e 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -31,6 +31,7 @@ #include <init/common.h> #include <interfaces/chain.h> #include <interfaces/init.h> +#include <interfaces/mining.h> #include <interfaces/node.h> #include <kernel/context.h> #include <key.h> @@ -109,12 +110,15 @@ #include <boost/signals2/signal.hpp> -#if ENABLE_ZMQ +#ifdef ENABLE_ZMQ #include <zmq/zmqabstractnotifier.h> #include <zmq/zmqnotificationinterface.h> #include <zmq/zmqrpc.h> #endif +using common::AmountErrMsg; +using common::InvalidPortErrMsg; +using common::ResolveErrMsg; using kernel::DumpMempool; using kernel::LoadMempool; using kernel::ValidationCacheSizes; @@ -133,6 +137,9 @@ using node::NodeContext; using node::ShouldPersistMempool; using node::ImportBlocks; using node::VerifyLoadedChainstate; +using util::Join; +using util::ReplaceAll; +using util::ToString; static constexpr bool DEFAULT_PROXYRANDOMIZE{true}; static constexpr bool DEFAULT_REST_ENABLE{false}; @@ -358,7 +365,7 @@ void Shutdown(NodeContext& node) client->stop(); } -#if ENABLE_ZMQ +#ifdef ENABLE_ZMQ if (g_zmq_notification_interface) { if (node.validation_signals) node.validation_signals->UnregisterValidationInterface(g_zmq_notification_interface.get()); g_zmq_notification_interface.reset(); @@ -403,7 +410,7 @@ static void HandleSIGHUP(int) static BOOL WINAPI consoleCtrlHandler(DWORD dwCtrlType) { if (!(*Assert(g_shutdown))()) { - LogPrintf("Error: failed to send shutdown signal on Ctrl-C\n"); + LogError("Failed to send shutdown signal on Ctrl-C\n"); return false; } Sleep(INFINITE); @@ -525,7 +532,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-maxreceivebuffer=<n>", strprintf("Maximum per-connection receive buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXRECEIVEBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxsendbuffer=<n>", strprintf("Maximum per-connection memory usage for the send buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXSENDBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxuploadtarget=<n>", strprintf("Tries to keep outbound traffic under the given target per 24h. Limit does not apply to peers with 'download' permission or blocks created within past week. 0 = no limit (default: %s). Optional suffix units [k|K|m|M|g|G|t|T] (default: M). Lowercase is 1000 base while uppercase is 1024 base", DEFAULT_MAX_UPLOAD_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); -#if HAVE_SOCKADDR_UN +#ifdef HAVE_SOCKADDR_UN argsman.AddArg("-onion=<ip:port|path>", "Use separate SOCKS5 proxy to reach peers via Tor onion services, set -noonion to disable (default: -proxy). May be a local file path prefixed with 'unix:'.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); #else argsman.AddArg("-onion=<ip:port>", "Use separate SOCKS5 proxy to reach peers via Tor onion services, set -noonion to disable (default: -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -538,7 +545,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-peerblockfilters", strprintf("Serve compact block filters to peers per BIP 157 (default: %u)", DEFAULT_PEERBLOCKFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-txreconciliation", strprintf("Enable transaction reconciliations per BIP 330 (default: %d)", DEFAULT_TXRECONCILIATION_ENABLE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-port=<port>", strprintf("Listen for connections on <port> (default: %u, testnet: %u, signet: %u, regtest: %u). Not relevant for I2P (see doc/i2p.md).", defaultChainParams->GetDefaultPort(), testnetChainParams->GetDefaultPort(), signetChainParams->GetDefaultPort(), regtestChainParams->GetDefaultPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); -#if HAVE_SOCKADDR_UN +#ifdef HAVE_SOCKADDR_UN argsman.AddArg("-proxy=<ip:port|path>", "Connect through SOCKS5 proxy, set -noproxy to disable (default: disabled). May be a local file path prefixed with 'unix:' if the proxy supports it.", ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_ELISION, OptionsCategory::CONNECTION); #else argsman.AddArg("-proxy=<ip:port>", "Connect through SOCKS5 proxy, set -noproxy to disable (default: disabled)", ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_ELISION, OptionsCategory::CONNECTION); @@ -572,7 +579,7 @@ void SetupServerArgs(ArgsManager& argsman) g_wallet_init_interface.AddWalletOptions(argsman); -#if ENABLE_ZMQ +#ifdef ENABLE_ZMQ argsman.AddArg("-zmqpubhashblock=<address>", "Enable publish hash block in <address>", ArgsManager::ALLOW_ANY, OptionsCategory::ZMQ); argsman.AddArg("-zmqpubhashtx=<address>", "Enable publish hash transaction in <address>", ArgsManager::ALLOW_ANY, OptionsCategory::ZMQ); argsman.AddArg("-zmqpubrawblock=<address>", "Enable publish raw block in <address>", ArgsManager::ALLOW_ANY, OptionsCategory::ZMQ); @@ -598,7 +605,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-checkblocks=<n>", strprintf("How many blocks to check at startup (default: %u, 0 = all)", DEFAULT_CHECKBLOCKS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checklevel=<n>", strprintf("How thorough the block verification of -checkblocks is: %s (0-4, default: %u)", Join(CHECKLEVEL_DOC, ", "), DEFAULT_CHECKLEVEL), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-checkblockindex", strprintf("Do a consistency check for the block tree, chainstate, and other validation data structures occasionally. (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-checkblockindex", strprintf("Do a consistency check for the block tree, chainstate, and other validation data structures every <n> operations. Use 0 to disable. (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checkaddrman=<n>", strprintf("Run addrman consistency checks every <n> operations. Use 0 to disable. (default: %u)", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checkmempool=<n>", strprintf("Run mempool consistency checks every <n> transactions. Use 0 to disable. (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checkpoints", strprintf("Enable rejection of any forks from the known historical chain until block %s (default: %u)", defaultChainParams->Checkpoints().GetHeight(), DEFAULT_CHECKPOINTS_ENABLED), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); @@ -834,7 +841,7 @@ std::set<BlockFilterType> g_enabled_filter_types; // Since LogPrintf may itself allocate memory, set the handler directly // to terminate first. std::set_new_handler(std::terminate); - LogPrintf("Error: Out of memory. Terminating.\n"); + LogError("Out of memory. Terminating.\n"); // The log was successful, terminate now. std::terminate(); @@ -1111,6 +1118,7 @@ bool AppInitLockDataDirectory() bool AppInitInterfaces(NodeContext& node) { node.chain = node.init->makeChain(); + node.mining = node.init->makeMining(); return true; } @@ -1169,9 +1177,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) scheduler.scheduleEvery([&args, &node]{ constexpr uint64_t min_disk_space = 50 << 20; // 50 MB if (!CheckDiskSpace(args.GetBlocksDirPath(), min_disk_space)) { - LogPrintf("Shutting down due to lack of disk space!\n"); + LogError("Shutting down due to lack of disk space!\n"); if (!(*Assert(node.shutdown))()) { - LogPrintf("Error: failed to send shutdown signal after disk space check\n"); + LogError("Failed to send shutdown signal after disk space check\n"); } } }, std::chrono::minutes{5}); @@ -1194,7 +1202,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) for (const auto& client : node.chain_clients) { client->registerRpcs(); } -#if ENABLE_ZMQ +#ifdef ENABLE_ZMQ RegisterZMQRPCCommands(tableRPC); #endif @@ -1319,7 +1327,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) std::string host_out; uint16_t port_out{0}; if (!SplitHostPort(socket_addr, port_out, host_out)) { -#if HAVE_SOCKADDR_UN +#ifdef HAVE_SOCKADDR_UN // Allow unix domain sockets for some options e.g. unix:/some/file/path if (!unix || socket_addr.find(ADDR_PREFIX_UNIX) != 0) { return InitError(InvalidPortErrMsg(arg, socket_addr)); @@ -1466,7 +1474,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) return InitError(ResolveErrMsg("externalip", strAddr)); } -#if ENABLE_ZMQ +#ifdef ENABLE_ZMQ g_zmq_notification_interface = CZMQNotificationInterface::Create( [&chainman = node.chainman](std::vector<uint8_t>& block, const CBlockIndex& index) { assert(chainman); @@ -1480,9 +1488,8 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // ********************************************************* Step 7: load block chain - node.notifications = std::make_unique<KernelNotifications>(*Assert(node.shutdown), node.exit_status); + node.notifications = std::make_unique<KernelNotifications>(*Assert(node.shutdown), node.exit_status, *Assert(node.warnings)); ReadNotificationArgs(args, *node.notifications); - bool fReindexChainState = args.GetBoolArg("-reindex-chainstate", false); ChainstateManager::Options chainman_opts{ .chainparams = chainparams, .datadir = args.GetDataDirNet(), @@ -1523,16 +1530,17 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) if (!result) { return InitError(util::ErrorString(result)); } - mempool_opts.check_ratio = std::clamp<int>(mempool_opts.check_ratio, 0, 1'000'000); - int64_t descendant_limit_bytes = mempool_opts.limits.descendant_size_vbytes * 40; - if (mempool_opts.max_size_bytes < 0 || mempool_opts.max_size_bytes < descendant_limit_bytes) { - return InitError(strprintf(_("-maxmempool must be at least %d MB"), std::ceil(descendant_limit_bytes / 1'000'000.0))); - } - LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", cache_sizes.coins * (1.0 / 1024 / 1024), mempool_opts.max_size_bytes * (1.0 / 1024 / 1024)); + bool do_reindex{args.GetBoolArg("-reindex", false)}; + const bool do_reindex_chainstate{args.GetBoolArg("-reindex-chainstate", false)}; for (bool fLoaded = false; !fLoaded && !ShutdownRequested(node);) { - node.mempool = std::make_unique<CTxMemPool>(mempool_opts); + bilingual_str mempool_error; + node.mempool = std::make_unique<CTxMemPool>(mempool_opts, mempool_error); + if (!mempool_error.empty()) { + return InitError(mempool_error); + } + LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", cache_sizes.coins * (1.0 / 1024 / 1024), mempool_opts.max_size_bytes * (1.0 / 1024 / 1024)); node.chainman = std::make_unique<ChainstateManager>(*Assert(node.shutdown), chainman_opts, blockman_opts); ChainstateManager& chainman = *node.chainman; @@ -1558,8 +1566,8 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) node::ChainstateLoadOptions options; options.mempool = Assert(node.mempool.get()); - options.reindex = chainman.m_blockman.m_reindexing; - options.reindex_chainstate = fReindexChainState; + options.wipe_block_tree_db = do_reindex; + options.wipe_chainstate_db = do_reindex || do_reindex_chainstate; options.prune = chainman.m_blockman.IsPruneMode(); options.check_blocks = args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); options.check_level = args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL); @@ -1576,7 +1584,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) try { return f(); } catch (const std::exception& e) { - LogPrintf("%s\n", e.what()); + LogError("%s\n", e.what()); return std::make_tuple(node::ChainstateLoadStatus::FAILURE, _("Error opening block database")); } }; @@ -1600,18 +1608,18 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) if (!fLoaded && !ShutdownRequested(node)) { // first suggest a reindex - if (!options.reindex) { + if (!do_reindex) { bool fRet = uiInterface.ThreadSafeQuestion( error + Untranslated(".\n\n") + _("Do you want to rebuild the block database now?"), error.original + ".\nPlease restart with -reindex or -reindex-chainstate to recover.", "", CClientUIInterface::MSG_ERROR | CClientUIInterface::BTN_ABORT); if (fRet) { - chainman.m_blockman.m_reindexing = true; + do_reindex = true; if (!Assert(node.shutdown)->reset()) { - LogPrintf("Internal error: failed to reset shutdown signal.\n"); + LogError("Internal error: failed to reset shutdown signal.\n"); } } else { - LogPrintf("Aborted block database rebuild. Exiting.\n"); + LogError("Aborted block database rebuild. Exiting.\n"); return false; } } else { @@ -1633,23 +1641,24 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) assert(!node.peerman); node.peerman = PeerManager::make(*node.connman, *node.addrman, node.banman.get(), chainman, - *node.mempool, peerman_opts); + *node.mempool, *node.warnings, + peerman_opts); validation_signals.RegisterValidationInterface(node.peerman.get()); // ********************************************************* Step 8: start indexers if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { - g_txindex = std::make_unique<TxIndex>(interfaces::MakeChain(node), cache_sizes.tx_index, false, chainman.m_blockman.m_reindexing); + g_txindex = std::make_unique<TxIndex>(interfaces::MakeChain(node), cache_sizes.tx_index, false, do_reindex); node.indexes.emplace_back(g_txindex.get()); } for (const auto& filter_type : g_enabled_filter_types) { - InitBlockFilterIndex([&]{ return interfaces::MakeChain(node); }, filter_type, cache_sizes.filter_index, false, chainman.m_blockman.m_reindexing); + InitBlockFilterIndex([&]{ return interfaces::MakeChain(node); }, filter_type, cache_sizes.filter_index, false, do_reindex); node.indexes.emplace_back(GetBlockFilterIndex(filter_type)); } if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) { - g_coin_stats_index = std::make_unique<CoinStatsIndex>(interfaces::MakeChain(node), /*cache_size=*/0, false, chainman.m_blockman.m_reindexing); + g_coin_stats_index = std::make_unique<CoinStatsIndex>(interfaces::MakeChain(node), /*cache_size=*/0, false, do_reindex); node.indexes.emplace_back(g_coin_stats_index.get()); } @@ -1668,7 +1677,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // if pruning, perform the initial blockstore prune // after any wallet rescanning has taken place. if (chainman.m_blockman.IsPruneMode()) { - if (!chainman.m_blockman.m_reindexing) { + if (chainman.m_blockman.m_blockfiles_indexed) { LOCK(cs_main); for (Chainstate* chainstate : chainman.GetAll()) { uiInterface.InitMessage(_("Pruning blockstore…").translated); @@ -1694,7 +1703,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) int chain_active_height = WITH_LOCK(cs_main, return chainman.ActiveChain().Height()); // On first startup, warn on low block storage space - if (!chainman.m_blockman.m_reindexing && !fReindexChainState && chain_active_height <= 1) { + if (!do_reindex && !do_reindex_chainstate && chain_active_height <= 1) { uint64_t assumed_chain_bytes{chainparams.AssumedBlockchainSize() * 1024 * 1024 * 1024}; uint64_t additional_bytes_needed{ chainman.m_blockman.IsPruneMode() ? @@ -1746,7 +1755,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) if (args.GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) { LogPrintf("Stopping after block import\n"); if (!(*Assert(node.shutdown))()) { - LogPrintf("Error: failed to send shutdown signal after finishing block import\n"); + LogError("Failed to send shutdown signal after finishing block import\n"); } return; } diff --git a/src/init/bitcoin-node.cpp b/src/init/bitcoin-node.cpp index 97b8dc1161..00a3822791 100644 --- a/src/init/bitcoin-node.cpp +++ b/src/init/bitcoin-node.cpp @@ -30,6 +30,7 @@ public: } std::unique_ptr<interfaces::Node> makeNode() override { return interfaces::MakeNode(m_node); } std::unique_ptr<interfaces::Chain> makeChain() override { return interfaces::MakeChain(m_node); } + std::unique_ptr<interfaces::Mining> makeMining() override { return interfaces::MakeMining(m_node); } std::unique_ptr<interfaces::WalletLoader> makeWalletLoader(interfaces::Chain& chain) override { return MakeWalletLoader(chain, *Assert(m_node.args)); diff --git a/src/init/bitcoin-qt.cpp b/src/init/bitcoin-qt.cpp index 3003a8fde1..5209c72973 100644 --- a/src/init/bitcoin-qt.cpp +++ b/src/init/bitcoin-qt.cpp @@ -6,6 +6,7 @@ #include <interfaces/chain.h> #include <interfaces/echo.h> #include <interfaces/init.h> +#include <interfaces/mining.h> #include <interfaces/node.h> #include <interfaces/wallet.h> #include <node/context.h> @@ -25,6 +26,7 @@ public: } std::unique_ptr<interfaces::Node> makeNode() override { return interfaces::MakeNode(m_node); } std::unique_ptr<interfaces::Chain> makeChain() override { return interfaces::MakeChain(m_node); } + std::unique_ptr<interfaces::Mining> makeMining() override { return interfaces::MakeMining(m_node); } std::unique_ptr<interfaces::WalletLoader> makeWalletLoader(interfaces::Chain& chain) override { return MakeWalletLoader(chain, *Assert(m_node.args)); diff --git a/src/init/bitcoind.cpp b/src/init/bitcoind.cpp index b5df764017..48be8831d2 100644 --- a/src/init/bitcoind.cpp +++ b/src/init/bitcoind.cpp @@ -6,6 +6,7 @@ #include <interfaces/chain.h> #include <interfaces/echo.h> #include <interfaces/init.h> +#include <interfaces/mining.h> #include <interfaces/node.h> #include <interfaces/wallet.h> #include <node/context.h> @@ -27,6 +28,7 @@ public: } std::unique_ptr<interfaces::Node> makeNode() override { return interfaces::MakeNode(m_node); } std::unique_ptr<interfaces::Chain> makeChain() override { return interfaces::MakeChain(m_node); } + std::unique_ptr<interfaces::Mining> makeMining() override { return interfaces::MakeMining(m_node); } std::unique_ptr<interfaces::WalletLoader> makeWalletLoader(interfaces::Chain& chain) override { return MakeWalletLoader(chain, *Assert(m_node.args)); diff --git a/src/init/common.cpp b/src/init/common.cpp index f473ee6d66..00ef879dc1 100644 --- a/src/init/common.cpp +++ b/src/init/common.cpp @@ -20,6 +20,8 @@ #include <string> #include <vector> +using util::SplitString; + namespace init { void AddLoggingArgs(ArgsManager& argsman) { diff --git a/src/interfaces/init.h b/src/interfaces/init.h index addc45aa26..094ead399d 100644 --- a/src/interfaces/init.h +++ b/src/interfaces/init.h @@ -7,6 +7,7 @@ #include <interfaces/chain.h> #include <interfaces/echo.h> +#include <interfaces/mining.h> #include <interfaces/node.h> #include <interfaces/wallet.h> @@ -32,6 +33,7 @@ public: virtual ~Init() = default; virtual std::unique_ptr<Node> makeNode() { return nullptr; } virtual std::unique_ptr<Chain> makeChain() { return nullptr; } + virtual std::unique_ptr<Mining> makeMining() { return nullptr; } virtual std::unique_ptr<WalletLoader> makeWalletLoader(Chain& chain) { return nullptr; } virtual std::unique_ptr<Echo> makeEcho() { return nullptr; } virtual Ipc* ipc() { return nullptr; } diff --git a/src/interfaces/mining.h b/src/interfaces/mining.h new file mode 100644 index 0000000000..b96881f67c --- /dev/null +++ b/src/interfaces/mining.h @@ -0,0 +1,82 @@ +// Copyright (c) 2024 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INTERFACES_MINING_H +#define BITCOIN_INTERFACES_MINING_H + +#include <optional> +#include <uint256.h> + +namespace node { +struct CBlockTemplate; +struct NodeContext; +} // namespace node + +class BlockValidationState; +class CBlock; +class CScript; + +namespace interfaces { + +//! Interface giving clients (RPC, Stratum v2 Template Provider in the future) +//! ability to create block templates. + +class Mining +{ +public: + virtual ~Mining() {} + + //! If this chain is exclusively used for testing + virtual bool isTestChain() = 0; + + //! Returns whether IBD is still in progress. + virtual bool isInitialBlockDownload() = 0; + + //! Returns the hash for the tip of this chain + virtual std::optional<uint256> getTipHash() = 0; + + /** + * Construct a new block template + * + * @param[in] script_pub_key the coinbase output + * @param[in] use_mempool set false to omit mempool transactions + * @returns a block template + */ + virtual std::unique_ptr<node::CBlockTemplate> createNewBlock(const CScript& script_pub_key, bool use_mempool = true) = 0; + /** + * Processes new block. A valid new block is automatically relayed to peers. + * + * @param[in] block The block we want to process. + * @param[out] new_block A boolean which is set to indicate if the block was first received via this call + * @returns If the block was processed, independently of block validity + */ + virtual bool processNewBlock(const std::shared_ptr<const CBlock>& block, bool* new_block) = 0; + + //! Return the number of transaction updates in the mempool, + //! used to decide whether to make a new block template. + virtual unsigned int getTransactionsUpdated() = 0; + + /** + * Check a block is completely valid from start to finish. + * Only works on top of our current best block. + * Does not check proof-of-work. + * + * @param[out] state details of why a block failed to validate + * @param[in] block the block to validate + * @param[in] check_merkle_root call CheckMerkleRoot() + * @returns false if any of the checks fail + */ + virtual bool testBlockValidity(BlockValidationState& state, const CBlock& block, bool check_merkle_root = true) = 0; + + //! Get internal node context. Useful for RPC and testing, + //! but not accessible across processes. + virtual node::NodeContext* context() { return nullptr; } +}; + +//! Return implementation of Mining interface. +std::unique_ptr<Mining> MakeMining(node::NodeContext& node); + +} // namespace interfaces + +#endif // BITCOIN_INTERFACES_MINING_H diff --git a/src/interfaces/node.h b/src/interfaces/node.h index aeb2612c07..2bb895dd47 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -30,10 +30,10 @@ class RPCTimerInterface; class UniValue; class Proxy; enum class SynchronizationState; -enum class TransactionError; struct CNodeStateStats; struct bilingual_str; namespace node { +enum class TransactionError; struct NodeContext; } // namespace node namespace wallet { @@ -208,7 +208,7 @@ public: virtual std::optional<Coin> getUnspentOutput(const COutPoint& output) = 0; //! Broadcast transaction. - virtual TransactionError broadcastTransaction(CTransactionRef tx, CAmount max_tx_fee, std::string& err_string) = 0; + virtual node::TransactionError broadcastTransaction(CTransactionRef tx, CAmount max_tx_fee, std::string& err_string) = 0; //! Get wallet loader. virtual WalletLoader& walletLoader() = 0; diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index c41f35829d..f7bcca58cf 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -6,13 +6,13 @@ #define BITCOIN_INTERFACES_WALLET_H #include <addresstype.h> +#include <common/signmessage.h> #include <consensus/amount.h> #include <interfaces/chain.h> #include <pubkey.h> #include <script/script.h> #include <support/allocators/secure.h> #include <util/fs.h> -#include <util/message.h> #include <util/result.h> #include <util/ui_change_type.h> @@ -30,9 +30,14 @@ class CFeeRate; class CKey; enum class FeeReason; enum class OutputType; -enum class TransactionError; struct PartiallySignedTransaction; struct bilingual_str; +namespace common { +enum class PSBTError; +} // namespace common +namespace node { +enum class TransactionError; +} // namespace node namespace wallet { class CCoinControl; class CWallet; @@ -202,7 +207,7 @@ public: int& num_blocks) = 0; //! Fill PSBT. - virtual TransactionError fillPSBT(int sighash_type, + virtual std::optional<common::PSBTError> fillPSBT(int sighash_type, bool sign, bool bip32derivs, size_t* n_signed, diff --git a/src/kernel/blockmanager_opts.h b/src/kernel/blockmanager_opts.h index 16072b669b..deeba7e318 100644 --- a/src/kernel/blockmanager_opts.h +++ b/src/kernel/blockmanager_opts.h @@ -24,7 +24,6 @@ struct BlockManagerOpts { bool fast_prune{false}; const fs::path blocks_dir; Notifications& notifications; - bool reindex{false}; }; } // namespace kernel diff --git a/src/kernel/chainparams.cpp b/src/kernel/chainparams.cpp index 26c261eba2..af02c6963b 100644 --- a/src/kernel/chainparams.cpp +++ b/src/kernel/chainparams.cpp @@ -29,7 +29,7 @@ static CBlock CreateGenesisBlock(const char* pszTimestamp, const CScript& genesisOutputScript, uint32_t nTime, uint32_t nNonce, uint32_t nBits, int32_t nVersion, const CAmount& genesisReward) { CMutableTransaction txNew; - txNew.nVersion = 1; + txNew.version = 1; txNew.vin.resize(1); txNew.vout.resize(1); txNew.vin[0].scriptSig = CScript() << 486604799 << CScriptNum(4) << std::vector<unsigned char>((const unsigned char*)pszTimestamp, (const unsigned char*)pszTimestamp + strlen(pszTimestamp)); @@ -542,3 +542,33 @@ std::unique_ptr<const CChainParams> CChainParams::TestNet() { return std::make_unique<const CTestNetParams>(); } + +std::vector<int> CChainParams::GetAvailableSnapshotHeights() const +{ + std::vector<int> heights; + heights.reserve(m_assumeutxo_data.size()); + + for (const auto& data : m_assumeutxo_data) { + heights.emplace_back(data.height); + } + return heights; +} + +std::optional<ChainType> GetNetworkForMagic(const MessageStartChars& message) +{ + const auto mainnet_msg = CChainParams::Main()->MessageStart(); + const auto testnet_msg = CChainParams::TestNet()->MessageStart(); + const auto regtest_msg = CChainParams::RegTest({})->MessageStart(); + const auto signet_msg = CChainParams::SigNet({})->MessageStart(); + + if (std::equal(message.begin(), message.end(), mainnet_msg.data())) { + return ChainType::MAIN; + } else if (std::equal(message.begin(), message.end(), testnet_msg.data())) { + return ChainType::TESTNET; + } else if (std::equal(message.begin(), message.end(), regtest_msg.data())) { + return ChainType::REGTEST; + } else if (std::equal(message.begin(), message.end(), signet_msg.data())) { + return ChainType::SIGNET; + } + return std::nullopt; +} diff --git a/src/kernel/chainparams.h b/src/kernel/chainparams.h index 7a5539bc71..05ebd07ec7 100644 --- a/src/kernel/chainparams.h +++ b/src/kernel/chainparams.h @@ -93,6 +93,7 @@ public: const Consensus::Params& GetConsensus() const { return consensus; } const MessageStartChars& MessageStart() const { return pchMessageStart; } uint16_t GetDefaultPort() const { return nDefaultPort; } + std::vector<int> GetAvailableSnapshotHeights() const; const CBlock& GenesisBlock() const { return genesis; } /** Default value for -checkmempool and -checkblockindex argument */ @@ -183,4 +184,6 @@ protected: ChainTxData chainTxData; }; +std::optional<ChainType> GetNetworkForMagic(const MessageStartChars& pchMessageStart); + #endif // BITCOIN_KERNEL_CHAINPARAMS_H diff --git a/src/kernel/chainstatemanager_opts.h b/src/kernel/chainstatemanager_opts.h index de5f78494a..076841c3c9 100644 --- a/src/kernel/chainstatemanager_opts.h +++ b/src/kernel/chainstatemanager_opts.h @@ -33,7 +33,7 @@ namespace kernel { struct ChainstateManagerOpts { const CChainParams& chainparams; fs::path datadir; - std::optional<bool> check_block_index{}; + std::optional<int32_t> check_block_index{}; bool checkpoints_enabled{DEFAULT_CHECKPOINTS_ENABLED}; //! If set, it will override the minimum work we will assume exists on some valid chain. std::optional<arith_uint256> minimum_chain_work{}; diff --git a/src/kernel/notifications_interface.h b/src/kernel/notifications_interface.h index 7283a88e86..8e090dd7db 100644 --- a/src/kernel/notifications_interface.h +++ b/src/kernel/notifications_interface.h @@ -16,6 +16,8 @@ namespace kernel { //! Result type for use with std::variant to indicate that an operation should be interrupted. struct Interrupted{}; +enum class Warning; + //! Simple result type for functions that need to propagate an interrupt status and don't have other return values. using InterruptResult = std::variant<std::monostate, Interrupted>; @@ -38,7 +40,8 @@ public: [[nodiscard]] virtual InterruptResult blockTip(SynchronizationState state, CBlockIndex& index) { return {}; } virtual void headerTip(SynchronizationState state, int64_t height, int64_t timestamp, bool presync) {} virtual void progress(const bilingual_str& title, int progress_percent, bool resume_possible) {} - virtual void warning(const bilingual_str& warning) {} + virtual void warningSet(Warning id, const bilingual_str& message) {} + virtual void warningUnset(Warning id) {} //! The flush error notification is sent to notify the user that an error //! occurred while flushing block data to disk. Kernel code may ignore flush diff --git a/src/kernel/warning.h b/src/kernel/warning.h new file mode 100644 index 0000000000..453f36c552 --- /dev/null +++ b/src/kernel/warning.h @@ -0,0 +1,14 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_KERNEL_WARNING_H +#define BITCOIN_KERNEL_WARNING_H + +namespace kernel { +enum class Warning { + UNKNOWN_NEW_RULES_ACTIVATED, + LARGE_WORK_INVALID_CHAIN, +}; +} // namespace kernel +#endif // BITCOIN_KERNEL_WARNING_H diff --git a/src/key.cpp b/src/key.cpp index e8458f2e3b..97d7821e74 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -166,12 +166,6 @@ void CKey::MakeNewKey(bool fCompressedIn) { fCompressed = fCompressedIn; } -bool CKey::Negate() -{ - assert(keydata); - return secp256k1_ec_seckey_negate(secp256k1_context_sign, keydata->data()); -} - CPrivKey CKey::GetPrivKey() const { assert(keydata); CPrivKey seckey; @@ -124,9 +124,6 @@ public: //! Generate a new private key using a cryptographic PRNG. void MakeNewKey(bool fCompressed); - //! Negate private key - bool Negate(); - /** * Convert the private key to a CPrivKey (serialized OpenSSL private key data). * This is expensive. diff --git a/src/leveldb/include/leveldb/status.h b/src/leveldb/include/leveldb/status.h index e3273144e4..68efe3001a 100644 --- a/src/leveldb/include/leveldb/status.h +++ b/src/leveldb/include/leveldb/status.h @@ -103,6 +103,8 @@ class LEVELDB_EXPORT Status { inline Status::Status(const Status& rhs) { state_ = (rhs.state_ == nullptr) ? nullptr : CopyState(rhs.state_); } + +// NOLINTBEGIN(bugprone-unhandled-self-assignment) inline Status& Status::operator=(const Status& rhs) { // The following condition catches both aliasing (when this == &rhs), // and the common case where both rhs and *this are ok. @@ -112,6 +114,8 @@ inline Status& Status::operator=(const Status& rhs) { } return *this; } +// NOLINTEND(bugprone-unhandled-self-assignment) + inline Status& Status::operator=(Status&& rhs) noexcept { std::swap(state_, rhs.state_); return *this; diff --git a/src/logging.cpp b/src/logging.cpp index 578650f856..a9fea433be 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -13,6 +13,10 @@ #include <map> #include <optional> +using util::Join; +using util::RemovePrefix; +using util::ToString; + const char * const DEFAULT_DEBUGLOGFILE = "debug.log"; constexpr auto MAX_USER_SETABLE_SEVERITY_LEVEL{BCLog::Level::Info}; diff --git a/src/logging.h b/src/logging.h index cfef65221f..fe6b7051ba 100644 --- a/src/logging.h +++ b/src/logging.h @@ -189,7 +189,7 @@ namespace BCLog { /** Returns a string with the log categories in alphabetical order. */ std::string LogCategoriesString() const { - return Join(LogCategoriesList(), ", ", [&](const LogCategory& i) { return i.category; }); + return util::Join(LogCategoriesList(), ", ", [&](const LogCategory& i) { return i.category; }); }; //! Returns a string with all user-selectable log levels. diff --git a/src/mapport.cpp b/src/mapport.cpp index 80670230c7..1920297be6 100644 --- a/src/mapport.cpp +++ b/src/mapport.cpp @@ -161,8 +161,11 @@ static bool ProcessUpnp() struct UPNPUrls urls; struct IGDdatas data; int r; - +#if MINIUPNPC_API_VERSION <= 17 r = UPNP_GetValidIGD(devlist, &urls, &data, lanaddr, sizeof(lanaddr)); +#else + r = UPNP_GetValidIGD(devlist, &urls, &data, lanaddr, sizeof(lanaddr), nullptr, 0); +#endif if (r == 1) { if (fDiscover) { diff --git a/src/minisketch/configure.ac b/src/minisketch/configure.ac index cd52d7f412..65a47b45c2 100644 --- a/src/minisketch/configure.ac +++ b/src/minisketch/configure.ac @@ -102,6 +102,7 @@ case $host in esac AX_CHECK_COMPILE_FLAG([-Wall],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wall"],,[[$CXXFLAG_WERROR]]) +AX_CHECK_COMPILE_FLAG([-Wundef], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wundef"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-fvisibility=hidden],[CXXFLAGS="$CXXFLAGS -fvisibility=hidden"],[],[$CXXFLAG_WERROR]) if test "x$use_ccache" != "xno"; then diff --git a/src/minisketch/include/minisketch.h b/src/minisketch/include/minisketch.h index 24d6b4e1c0..b0571d2788 100644 --- a/src/minisketch/include/minisketch.h +++ b/src/minisketch/include/minisketch.h @@ -239,7 +239,7 @@ public: /** Make this Minisketch a clone of the specified one. */ Minisketch& operator=(const Minisketch& sketch) noexcept { - if (sketch.m_minisketch) { + if (this != &sketch && sketch.m_minisketch) { m_minisketch = std::unique_ptr<minisketch, Deleter>(minisketch_clone(sketch.m_minisketch.get())); } return *this; diff --git a/src/minisketch/src/false_positives.h b/src/minisketch/src/false_positives.h index 44ebb3e94c..9d0358997f 100644 --- a/src/minisketch/src/false_positives.h +++ b/src/minisketch/src/false_positives.h @@ -81,7 +81,8 @@ uint64_t BaseFPBits(uint32_t bits, uint32_t capacity) { size_t ComputeCapacity(uint32_t bits, size_t max_elements, uint32_t fpbits) { if (bits == 0) return 0; - uint64_t base_fpbits = BaseFPBits(bits, max_elements); + if (max_elements > 0xffffffff) return max_elements; + uint64_t base_fpbits = BaseFPBits(bits, static_cast<uint32_t>(max_elements)); // The fpbits provided by the base max_elements==capacity case are sufficient. if (base_fpbits >= fpbits) return max_elements; // Otherwise, increment capacity by ceil(fpbits / bits) beyond that. @@ -90,6 +91,7 @@ size_t ComputeCapacity(uint32_t bits, size_t max_elements, uint32_t fpbits) { size_t ComputeMaxElements(uint32_t bits, size_t capacity, uint32_t fpbits) { if (bits == 0) return 0; + if (capacity > 0xffffffff) return capacity; // Start with max_elements=capacity, and decrease max_elements until the corresponding capacity is capacity. size_t max_elements = capacity; while (true) { diff --git a/src/minisketch/src/int_utils.h b/src/minisketch/src/int_utils.h index 2b3d8cb402..a6b89cd63c 100644 --- a/src/minisketch/src/int_utils.h +++ b/src/minisketch/src/int_utils.h @@ -159,7 +159,7 @@ static inline int CountBits(I val, int max) { } if (!ret) return 0; return index + 1; -#elif HAVE_CLZ +#elif defined(HAVE_CLZ) (void)max; if (val == 0) return 0; if (std::numeric_limits<unsigned>::digits >= std::numeric_limits<I>::digits) { @@ -210,7 +210,7 @@ public: static constexpr inline int TopBits(I val) { static_assert(Count > 0, "BitsInt::TopBits needs Count > 0"); static_assert(Count <= BITS, "BitsInt::TopBits needs Offset <= BITS"); - return val >> (BITS - Count); + return static_cast<int>(val >> (BITS - Count)); } static inline constexpr I CondXorWith(I val, bool cond, I v) { diff --git a/src/minisketch/src/minisketch.cpp b/src/minisketch/src/minisketch.cpp index d003fdf755..2e45409243 100644 --- a/src/minisketch/src/minisketch.cpp +++ b/src/minisketch/src/minisketch.cpp @@ -468,7 +468,7 @@ size_t minisketch_merge(minisketch* sketch, const minisketch* other_sketch) { ssize_t minisketch_decode(const minisketch* sketch, size_t max_elements, uint64_t* output) { const Sketch* s = (const Sketch*)sketch; s->Check(); - return s->Decode(max_elements, output); + return s->Decode(static_cast<int>(max_elements), output); } void minisketch_set_seed(minisketch* sketch, uint64_t seed) { diff --git a/src/minisketch/src/sketch.h b/src/minisketch/src/sketch.h index 3e9bad793d..662b4e982f 100644 --- a/src/minisketch/src/sketch.h +++ b/src/minisketch/src/sketch.h @@ -29,7 +29,7 @@ public: virtual ~Sketch() {} virtual size_t Syndromes() const = 0; - virtual void Init(int syndromes) = 0; + virtual void Init(size_t syndromes) = 0; virtual void Add(uint64_t element) = 0; virtual void Serialize(unsigned char*) const = 0; virtual void Deserialize(const unsigned char*) = 0; diff --git a/src/minisketch/src/sketch_impl.h b/src/minisketch/src/sketch_impl.h index 4547b742f2..c357f0e823 100644 --- a/src/minisketch/src/sketch_impl.h +++ b/src/minisketch/src/sketch_impl.h @@ -92,7 +92,8 @@ template<typename F> void Sqr(std::vector<typename F::Elem>& poly, const F& field) { if (poly.size() == 0) return; poly.resize(poly.size() * 2 - 1); - for (int x = poly.size() - 1; x >= 0; --x) { + for (size_t i = 0; i < poly.size(); ++i) { + auto x = poly.size() - i - 1; poly[x] = (x & 1) ? 0 : field.Sqr(poly[x / 2]); } } @@ -217,7 +218,7 @@ bool RecFindRoots(std::vector<std::vector<typename F::Elem>>& stack, size_t pos, } if (fully_factorizable) { - // Every succesful iteration of this algorithm splits the input + // Every successful iteration of this algorithm splits the input // polynomial further into buckets, each corresponding to a subset // of 2^(BITS-depth) roots. If after depth splits the degree of // the polynomial is >= 2^(BITS-depth), something is wrong. @@ -297,7 +298,7 @@ std::vector<typename F::Elem> BerlekampMassey(const std::vector<typename F::Elem auto discrepancy = syndromes[n]; for (size_t i = 1; i < current.size(); ++i) discrepancy ^= table[n - i](current[i]); if (discrepancy != 0) { - int x = n + 1 - (current.size() - 1) - (prev.size() - 1); + int x = static_cast<int>(n + 1 - (current.size() - 1) - (prev.size() - 1)); if (!b_have_inv) { b_inv = field.Inv(b); b_have_inv = true; @@ -366,7 +367,7 @@ public: } size_t Syndromes() const override { return m_syndromes.size(); } - void Init(int count) override { m_syndromes.assign(count, 0); } + void Init(size_t count) override { m_syndromes.assign(count, 0); } void Add(uint64_t val) override { @@ -405,7 +406,7 @@ public: for (const auto& root : roots) { *(out++) = m_field.ToUint64(root); } - return roots.size(); + return static_cast<int>(roots.size()); } size_t Merge(const Sketch* other_sketch) override diff --git a/src/net.cpp b/src/net.cpp index de974f39cb..990c58ee3d 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -3029,7 +3029,7 @@ bool CConnman::BindListenPort(const CService& addrBind, bilingual_str& strError, return false; } - std::unique_ptr<Sock> sock = CreateSock(addrBind.GetSAFamily()); + std::unique_ptr<Sock> sock = CreateSock(addrBind.GetSAFamily(), SOCK_STREAM, IPPROTO_TCP); if (!sock) { strError = strprintf(Untranslated("Couldn't open socket for incoming connections (socket returned error %s)"), NetworkErrorString(WSAGetLastError())); LogPrintLevel(BCLog::NET, BCLog::Level::Error, "%s\n", strError.original); diff --git a/src/net_permissions.cpp b/src/net_permissions.cpp index b01b2f643d..8f0042c141 100644 --- a/src/net_permissions.cpp +++ b/src/net_permissions.cpp @@ -2,12 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/messages.h> #include <common/system.h> #include <net_permissions.h> #include <netbase.h> -#include <util/error.h> #include <util/translation.h> +using common::ResolveErrMsg; + const std::vector<std::string> NET_PERMISSIONS_DOC{ "bloomfilter (allow requesting BIP37 filtered blocks and transactions)", "noban (do not ban for misbehavior; implies download)", diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 6374cb52c1..89b9488584 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -25,6 +25,7 @@ #include <node/blockstorage.h> #include <node/timeoffsets.h> #include <node/txreconciliation.h> +#include <node/warnings.h> #include <policy/fees.h> #include <policy/policy.h> #include <policy/settings.h> @@ -130,8 +131,6 @@ static constexpr double BLOCK_DOWNLOAD_TIMEOUT_BASE = 1; static constexpr double BLOCK_DOWNLOAD_TIMEOUT_PER_PEER = 0.5; /** Maximum number of headers to announce when relaying blocks with headers message.*/ static const unsigned int MAX_BLOCKS_TO_ANNOUNCE = 8; -/** Maximum number of unconnecting headers announcements before DoS score */ -static const int MAX_NUM_UNCONNECTING_HEADERS_MSGS = 10; /** Minimum blocks required to signal NODE_NETWORK_LIMITED */ static const unsigned int NODE_NETWORK_LIMITED_MIN_BLOCKS = 288; /** Window, in blocks, for connecting to NODE_NETWORK_LIMITED peers */ @@ -225,8 +224,6 @@ struct Peer { /** Protects misbehavior data members */ Mutex m_misbehavior_mutex; - /** Accumulated misbehavior score for this peer */ - int m_misbehavior_score GUARDED_BY(m_misbehavior_mutex){0}; /** Whether this peer should be disconnected and marked as discouraged (unless it has NetPermissionFlags::NoBan permission). */ bool m_should_discourage GUARDED_BY(m_misbehavior_mutex){false}; @@ -382,9 +379,6 @@ struct Peer { /** Whether we've sent our peer a sendheaders message. **/ std::atomic<bool> m_sent_sendheaders{false}; - /** Length of current-streak of unconnecting headers announcements */ - int m_num_unconnecting_headers_msgs GUARDED_BY(NetEventsInterface::g_msgproc_mutex){0}; - /** When to potentially disconnect peer for stalling headers download */ std::chrono::microseconds m_headers_sync_timeout GUARDED_BY(NetEventsInterface::g_msgproc_mutex){0us}; @@ -489,7 +483,7 @@ class PeerManagerImpl final : public PeerManager public: PeerManagerImpl(CConnman& connman, AddrMan& addrman, BanMan* banman, ChainstateManager& chainman, - CTxMemPool& pool, Options opts); + CTxMemPool& pool, node::Warnings& warnings, Options opts); /** Overridden from CValidationInterface. */ void BlockConnected(ChainstateRole role, const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected) override @@ -526,7 +520,7 @@ public: m_best_height = height; m_best_block_time = time; }; - void UnitTestMisbehaving(NodeId peer_id, int howmuch) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex) { Misbehaving(*Assert(GetPeerRef(peer_id)), howmuch, ""); }; + void UnitTestMisbehaving(NodeId peer_id) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex) { Misbehaving(*Assert(GetPeerRef(peer_id)), ""); }; void ProcessMessage(CNode& pfrom, const std::string& msg_type, DataStream& vRecv, const std::chrono::microseconds time_received, const std::atomic<bool>& interruptMsgProc) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, !m_recent_confirmed_transactions_mutex, !m_most_recent_block_mutex, !m_headers_presync_mutex, g_msgproc_mutex); @@ -551,11 +545,9 @@ private: * May return an empty shared_ptr if the Peer object can't be found. */ PeerRef RemovePeer(NodeId id) EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); - /** - * Increment peer's misbehavior score. If the new value >= DISCOURAGEMENT_THRESHOLD, mark the node - * to be discouraged, meaning the peer might be disconnected and added to the discouragement filter. - */ - void Misbehaving(Peer& peer, int howmuch, const std::string& message); + /** Mark a peer as misbehaving, which will cause it to be disconnected and its + * address discouraged. */ + void Misbehaving(Peer& peer, const std::string& message); /** * Potentially mark a node discouraged based on the contents of a BlockValidationState object @@ -564,19 +556,15 @@ private: * punish peers differently depending on whether the data was provided in a compact * block message or not. If the compact block had a valid header, but contained invalid * txs, the peer should not be punished. See BIP 152. - * - * @return Returns true if the peer was punished (probably disconnected) */ - bool MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& state, + void MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& state, bool via_compact_block, const std::string& message = "") EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); /** * Potentially disconnect and discourage a node based on the contents of a TxValidationState object - * - * @return Returns true if the peer was punished (probably disconnected) */ - bool MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state) + void MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state) EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); /** Maybe disconnect a peer and discourage future connections from its address. @@ -667,10 +655,10 @@ private: bool CheckHeadersPoW(const std::vector<CBlockHeader>& headers, const Consensus::Params& consensusParams, Peer& peer); /** Calculate an anti-DoS work threshold for headers chains */ arith_uint256 GetAntiDoSWorkThreshold(); - /** Deal with state tracking and headers sync for peers that send the - * occasional non-connecting header (this can happen due to BIP 130 headers + /** Deal with state tracking and headers sync for peers that send + * non-connecting headers (this can happen due to BIP 130 headers * announcements for blocks interacting with the 2hr (MAX_FUTURE_BLOCK_TIME) rule). */ - void HandleFewUnconnectingHeaders(CNode& pfrom, Peer& peer, const std::vector<CBlockHeader>& headers) EXCLUSIVE_LOCKS_REQUIRED(g_msgproc_mutex); + void HandleUnconnectingHeaders(CNode& pfrom, Peer& peer, const std::vector<CBlockHeader>& headers) EXCLUSIVE_LOCKS_REQUIRED(g_msgproc_mutex); /** Return true if the headers connect to each other, false otherwise */ bool CheckHeadersAreContinuous(const std::vector<CBlockHeader>& headers) const; /** Try to continue a low-work headers sync that has already begun. @@ -790,7 +778,8 @@ private: /** Next time to check for stale tip */ std::chrono::seconds m_stale_tip_check_time GUARDED_BY(cs_main){0s}; - TimeOffsets m_outbound_time_offsets; + node::Warnings& m_warnings; + TimeOffsets m_outbound_time_offsets{m_warnings}; const Options m_opts; @@ -1175,7 +1164,7 @@ private: void PushAddress(Peer& peer, const CAddress& addr) EXCLUSIVE_LOCKS_REQUIRED(g_msgproc_mutex); }; -const CNodeState* PeerManagerImpl::State(NodeId pnode) const EXCLUSIVE_LOCKS_REQUIRED(cs_main) +const CNodeState* PeerManagerImpl::State(NodeId pnode) const { std::map<NodeId, CNodeState>::const_iterator it = m_node_states.find(pnode); if (it == m_node_states.end()) @@ -1183,7 +1172,7 @@ const CNodeState* PeerManagerImpl::State(NodeId pnode) const EXCLUSIVE_LOCKS_REQ return &it->second; } -CNodeState* PeerManagerImpl::State(NodeId pnode) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +CNodeState* PeerManagerImpl::State(NodeId pnode) { return const_cast<CNodeState*>(std::as_const(*this).State(pnode)); } @@ -1716,7 +1705,6 @@ void PeerManagerImpl::ReattemptInitialBroadcast(CScheduler& scheduler) void PeerManagerImpl::FinalizeNode(const CNode& node) { NodeId nodeid = node.GetId(); - int misbehavior{0}; { LOCK(cs_main); { @@ -1727,7 +1715,6 @@ void PeerManagerImpl::FinalizeNode(const CNode& node) // destructed. PeerRef peer = RemovePeer(nodeid); assert(peer != nullptr); - misbehavior = WITH_LOCK(peer->m_misbehavior_mutex, return peer->m_misbehavior_score); m_wtxid_relay_peers -= peer->m_wtxid_relay; assert(m_wtxid_relay_peers >= 0); } @@ -1770,7 +1757,7 @@ void PeerManagerImpl::FinalizeNode(const CNode& node) assert(m_orphanage.Size() == 0); } } // cs_main - if (node.fSuccessfullyConnected && misbehavior == 0 && + if (node.fSuccessfullyConnected && !node.IsBlockOnlyConn() && !node.IsInboundConn()) { // Only change visible addrman state for full outbound peers. We don't // call Connected() for feeler connections since they don't have @@ -1891,28 +1878,16 @@ void PeerManagerImpl::AddToCompactExtraTransactions(const CTransactionRef& tx) vExtraTxnForCompactIt = (vExtraTxnForCompactIt + 1) % m_opts.max_extra_txs; } -void PeerManagerImpl::Misbehaving(Peer& peer, int howmuch, const std::string& message) +void PeerManagerImpl::Misbehaving(Peer& peer, const std::string& message) { - assert(howmuch > 0); - LOCK(peer.m_misbehavior_mutex); - const int score_before{peer.m_misbehavior_score}; - peer.m_misbehavior_score += howmuch; - const int score_now{peer.m_misbehavior_score}; const std::string message_prefixed = message.empty() ? "" : (": " + message); - std::string warning; - - if (score_now >= DISCOURAGEMENT_THRESHOLD && score_before < DISCOURAGEMENT_THRESHOLD) { - warning = " DISCOURAGE THRESHOLD EXCEEDED"; - peer.m_should_discourage = true; - } - - LogPrint(BCLog::NET, "Misbehaving: peer=%d (%d -> %d)%s%s\n", - peer.m_id, score_before, score_now, warning, message_prefixed); + peer.m_should_discourage = true; + LogPrint(BCLog::NET, "Misbehaving: peer=%d%s\n", peer.m_id, message_prefixed); } -bool PeerManagerImpl::MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& state, +void PeerManagerImpl::MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& state, bool via_compact_block, const std::string& message) { PeerRef peer{GetPeerRef(nodeid)}; @@ -1927,8 +1902,8 @@ bool PeerManagerImpl::MaybePunishNodeForBlock(NodeId nodeid, const BlockValidati case BlockValidationResult::BLOCK_CONSENSUS: case BlockValidationResult::BLOCK_MUTATED: if (!via_compact_block) { - if (peer) Misbehaving(*peer, 100, message); - return true; + if (peer) Misbehaving(*peer, message); + return; } break; case BlockValidationResult::BLOCK_CACHED_INVALID: @@ -1942,21 +1917,20 @@ bool PeerManagerImpl::MaybePunishNodeForBlock(NodeId nodeid, const BlockValidati // Discourage outbound (but not inbound) peers if on an invalid chain. // Exempt HB compact block peers. Manual connections are always protected from discouragement. if (!via_compact_block && !node_state->m_is_inbound) { - if (peer) Misbehaving(*peer, 100, message); - return true; + if (peer) Misbehaving(*peer, message); + return; } break; } case BlockValidationResult::BLOCK_INVALID_HEADER: case BlockValidationResult::BLOCK_CHECKPOINT: case BlockValidationResult::BLOCK_INVALID_PREV: - if (peer) Misbehaving(*peer, 100, message); - return true; + if (peer) Misbehaving(*peer, message); + return; // Conflicting (but not necessarily invalid) data or different policy: case BlockValidationResult::BLOCK_MISSING_PREV: - // TODO: Handle this much more gracefully (10 DoS points is super arbitrary) - if (peer) Misbehaving(*peer, 10, message); - return true; + if (peer) Misbehaving(*peer, message); + return; case BlockValidationResult::BLOCK_RECENT_CONSENSUS_CHANGE: case BlockValidationResult::BLOCK_TIME_FUTURE: break; @@ -1964,10 +1938,9 @@ bool PeerManagerImpl::MaybePunishNodeForBlock(NodeId nodeid, const BlockValidati if (message != "") { LogPrint(BCLog::NET, "peer=%d: %s\n", nodeid, message); } - return false; } -bool PeerManagerImpl::MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state) +void PeerManagerImpl::MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state) { PeerRef peer{GetPeerRef(nodeid)}; switch (state.GetResult()) { @@ -1975,8 +1948,8 @@ bool PeerManagerImpl::MaybePunishNodeForTx(NodeId nodeid, const TxValidationStat break; // The node is providing invalid data: case TxValidationResult::TX_CONSENSUS: - if (peer) Misbehaving(*peer, 100, ""); - return true; + if (peer) Misbehaving(*peer, ""); + return; // Conflicting (but not necessarily invalid) data or different policy: case TxValidationResult::TX_RECENT_CONSENSUS_CHANGE: case TxValidationResult::TX_INPUTS_NOT_STANDARD: @@ -1992,7 +1965,6 @@ bool PeerManagerImpl::MaybePunishNodeForTx(NodeId nodeid, const TxValidationStat case TxValidationResult::TX_UNKNOWN: break; } - return false; } bool PeerManagerImpl::BlockRequestAllowed(const CBlockIndex* pindex) @@ -2042,14 +2014,14 @@ std::optional<std::string> PeerManagerImpl::FetchBlock(NodeId peer_id, const CBl std::unique_ptr<PeerManager> PeerManager::make(CConnman& connman, AddrMan& addrman, BanMan* banman, ChainstateManager& chainman, - CTxMemPool& pool, Options opts) + CTxMemPool& pool, node::Warnings& warnings, Options opts) { - return std::make_unique<PeerManagerImpl>(connman, addrman, banman, chainman, pool, opts); + return std::make_unique<PeerManagerImpl>(connman, addrman, banman, chainman, pool, warnings, opts); } PeerManagerImpl::PeerManagerImpl(CConnman& connman, AddrMan& addrman, BanMan* banman, ChainstateManager& chainman, - CTxMemPool& pool, Options opts) + CTxMemPool& pool, node::Warnings& warnings, Options opts) : m_rng{opts.deterministic_rng}, m_fee_filter_rounder{CFeeRate{DEFAULT_MIN_RELAY_TX_FEE}, m_rng}, m_chainparams(chainman.GetParams()), @@ -2058,6 +2030,7 @@ PeerManagerImpl::PeerManagerImpl(CConnman& connman, AddrMan& addrman, m_banman(banman), m_chainman(chainman), m_mempool(pool), + m_warnings{warnings}, m_opts{opts} { // While Erlay support is incomplete, it must be enabled explicitly via -txreconciliation. @@ -2676,7 +2649,7 @@ void PeerManagerImpl::SendBlockTransactions(CNode& pfrom, Peer& peer, const CBlo BlockTransactions resp(req); for (size_t i = 0; i < req.indexes.size(); i++) { if (req.indexes[i] >= block.vtx.size()) { - Misbehaving(peer, 100, "getblocktxn with out-of-bounds tx indices"); + Misbehaving(peer, "getblocktxn with out-of-bounds tx indices"); return; } resp.txn[i] = block.vtx[req.indexes[i]]; @@ -2689,13 +2662,13 @@ bool PeerManagerImpl::CheckHeadersPoW(const std::vector<CBlockHeader>& headers, { // Do these headers have proof-of-work matching what's claimed? if (!HasValidProofOfWork(headers, consensusParams)) { - Misbehaving(peer, 100, "header with invalid proof of work"); + Misbehaving(peer, "header with invalid proof of work"); return false; } // Are these headers connected to each other? if (!CheckHeadersAreContinuous(headers)) { - Misbehaving(peer, 20, "non-continuous headers sequence"); + Misbehaving(peer, "non-continuous headers sequence"); return false; } return true; @@ -2719,37 +2692,24 @@ arith_uint256 PeerManagerImpl::GetAntiDoSWorkThreshold() * announcement. * * We'll send a getheaders message in response to try to connect the chain. - * - * The peer can send up to MAX_NUM_UNCONNECTING_HEADERS_MSGS in a row that - * don't connect before given DoS points. - * - * Once a headers message is received that is valid and does connect, - * m_num_unconnecting_headers_msgs gets reset back to 0. */ -void PeerManagerImpl::HandleFewUnconnectingHeaders(CNode& pfrom, Peer& peer, +void PeerManagerImpl::HandleUnconnectingHeaders(CNode& pfrom, Peer& peer, const std::vector<CBlockHeader>& headers) { - peer.m_num_unconnecting_headers_msgs++; // Try to fill in the missing headers. const CBlockIndex* best_header{WITH_LOCK(cs_main, return m_chainman.m_best_header)}; if (MaybeSendGetHeaders(pfrom, GetLocator(best_header), peer)) { - LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, m_num_unconnecting_headers_msgs=%d)\n", + LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d)\n", headers[0].GetHash().ToString(), headers[0].hashPrevBlock.ToString(), best_header->nHeight, - pfrom.GetId(), peer.m_num_unconnecting_headers_msgs); + pfrom.GetId()); } // Set hashLastUnknownBlock for this peer, so that if we // eventually get the headers - even from a different peer - // we can use this peer to download. WITH_LOCK(cs_main, UpdateBlockAvailability(pfrom.GetId(), headers.back().GetHash())); - - // The peer may just be broken, so periodically assign DoS points if this - // condition persists. - if (peer.m_num_unconnecting_headers_msgs % MAX_NUM_UNCONNECTING_HEADERS_MSGS == 0) { - Misbehaving(peer, 20, strprintf("%d non-connecting headers", peer.m_num_unconnecting_headers_msgs)); - } } bool PeerManagerImpl::CheckHeadersAreContinuous(const std::vector<CBlockHeader>& headers) const @@ -2768,25 +2728,21 @@ bool PeerManagerImpl::IsContinuationOfLowWorkHeadersSync(Peer& peer, CNode& pfro { if (peer.m_headers_sync) { auto result = peer.m_headers_sync->ProcessNextHeaders(headers, headers.size() == MAX_HEADERS_RESULTS); + // If it is a valid continuation, we should treat the existing getheaders request as responded to. + if (result.success) peer.m_last_getheaders_timestamp = {}; if (result.request_more) { auto locator = peer.m_headers_sync->NextHeadersRequestLocator(); // If we were instructed to ask for a locator, it should not be empty. Assume(!locator.vHave.empty()); + // We can only be instructed to request more if processing was successful. + Assume(result.success); if (!locator.vHave.empty()) { // It should be impossible for the getheaders request to fail, - // because we should have cleared the last getheaders timestamp - // when processing the headers that triggered this call. But - // it may be possible to bypass this via compactblock - // processing, so check the result before logging just to be - // safe. + // because we just cleared the last getheaders timestamp. bool sent_getheaders = MaybeSendGetHeaders(pfrom, locator, peer); - if (sent_getheaders) { - LogPrint(BCLog::NET, "more getheaders (from %s) to peer=%d\n", - locator.vHave.front().ToString(), pfrom.GetId()); - } else { - LogPrint(BCLog::NET, "error sending next getheaders (from %s) to continue sync with peer=%d\n", - locator.vHave.front().ToString(), pfrom.GetId()); - } + Assume(sent_getheaders); + LogPrint(BCLog::NET, "more getheaders (from %s) to peer=%d\n", + locator.vHave.front().ToString(), pfrom.GetId()); } } @@ -2995,11 +2951,6 @@ void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const Peer& peer, c void PeerManagerImpl::UpdatePeerStateForReceivedHeaders(CNode& pfrom, Peer& peer, const CBlockIndex& last_header, bool received_new_header, bool may_have_more_headers) { - if (peer.m_num_unconnecting_headers_msgs > 0) { - LogPrint(BCLog::NET, "peer=%d: resetting m_num_unconnecting_headers_msgs (%d -> 0)\n", pfrom.GetId(), peer.m_num_unconnecting_headers_msgs); - } - peer.m_num_unconnecting_headers_msgs = 0; - LOCK(cs_main); CNodeState *nodestate = State(pfrom.GetId()); @@ -3065,6 +3016,9 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, LOCK(m_headers_presync_mutex); m_headers_presync_stats.erase(pfrom.GetId()); } + // A headers message with no headers cannot be an announcement, so assume + // it is a response to our last getheaders request, if there is one. + peer.m_last_getheaders_timestamp = {}; return; } @@ -3118,17 +3072,18 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, bool headers_connect_blockindex{chain_start_header != nullptr}; if (!headers_connect_blockindex) { - if (nCount <= MAX_BLOCKS_TO_ANNOUNCE) { - // If this looks like it could be a BIP 130 block announcement, use - // special logic for handling headers that don't connect, as this - // could be benign. - HandleFewUnconnectingHeaders(pfrom, peer, headers); - } else { - Misbehaving(peer, 10, "invalid header received"); - } + // This could be a BIP 130 block announcement, use + // special logic for handling headers that don't connect, as this + // could be benign. + HandleUnconnectingHeaders(pfrom, peer, headers); return; } + // If headers connect, assume that this is in response to any outstanding getheaders + // request we may have sent, and clear out the time of our last request. Non-connecting + // headers cannot be a response to a getheaders request. + peer.m_last_getheaders_timestamp = {}; + // If the headers we received are already in memory and an ancestor of // m_best_header or our tip, skip anti-DoS checks. These headers will not // use any more memory (and we are not leaking information that could be @@ -3646,7 +3601,7 @@ void PeerManagerImpl::ProcessCompactBlockTxns(CNode& pfrom, Peer& peer, const Bl ReadStatus status = partialBlock.FillBlock(*pblock, block_transactions.txn); if (status == READ_STATUS_INVALID) { RemoveBlockRequest(block_transactions.blockhash, pfrom.GetId()); // Reset in-flight state in case Misbehaving does not result in a disconnect - Misbehaving(peer, 100, "invalid compact block/non-matching block transactions"); + Misbehaving(peer, "invalid compact block/non-matching block transactions"); return; } else if (status == READ_STATUS_FAILED) { if (first_in_flight) { @@ -4130,7 +4085,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (vAddr.size() > MAX_ADDR_TO_SEND) { - Misbehaving(*peer, 20, strprintf("%s message size = %u", msg_type, vAddr.size())); + Misbehaving(*peer, strprintf("%s message size = %u", msg_type, vAddr.size())); return; } @@ -4212,7 +4167,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, vRecv >> vInv; if (vInv.size() > MAX_INV_SZ) { - Misbehaving(*peer, 20, strprintf("inv message size = %u", vInv.size())); + Misbehaving(*peer, strprintf("inv message size = %u", vInv.size())); return; } @@ -4304,7 +4259,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, vRecv >> vInv; if (vInv.size() > MAX_INV_SZ) { - Misbehaving(*peer, 20, strprintf("getdata message size = %u", vInv.size())); + Misbehaving(*peer, strprintf("getdata message size = %u", vInv.size())); return; } @@ -4844,7 +4799,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, ReadStatus status = partialBlock.InitData(cmpctblock, vExtraTxnForCompact); if (status == READ_STATUS_INVALID) { RemoveBlockRequest(pindex->GetBlockHash(), pfrom.GetId()); // Reset in-flight state in case Misbehaving does not result in a disconnect - Misbehaving(*peer, 100, "invalid compact block"); + Misbehaving(*peer, "invalid compact block"); return; } else if (status == READ_STATUS_FAILED) { if (first_in_flight) { @@ -4984,16 +4939,12 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } - // Assume that this is in response to any outstanding getheaders - // request we may have sent, and clear out the time of our last request - peer->m_last_getheaders_timestamp = {}; - std::vector<CBlockHeader> headers; // Bypass the normal CBlock deserialization, as we don't want to risk deserializing 2000 full blocks. unsigned int nCount = ReadCompactSize(vRecv); if (nCount > MAX_HEADERS_RESULTS) { - Misbehaving(*peer, 20, strprintf("headers message size = %u", nCount)); + Misbehaving(*peer, strprintf("headers message size = %u", nCount)); return; } headers.resize(nCount); @@ -5040,7 +4991,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (prev_block && IsBlockMutated(/*block=*/*pblock, /*check_witness_root=*/DeploymentActiveAfter(prev_block, m_chainman, Consensus::DEPLOYMENT_SEGWIT))) { LogDebug(BCLog::NET, "Received mutated block from peer=%d\n", peer->m_id); - Misbehaving(*peer, 100, "mutated block"); + Misbehaving(*peer, "mutated block"); WITH_LOCK(cs_main, RemoveBlockRequest(pblock->GetHash(), peer->m_id)); return; } @@ -5221,7 +5172,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (!filter.IsWithinSizeConstraints()) { // There is no excuse for sending a too-large filter - Misbehaving(*peer, 100, "too-large bloom filter"); + Misbehaving(*peer, "too-large bloom filter"); } else if (auto tx_relay = peer->GetTxRelay(); tx_relay != nullptr) { { LOCK(tx_relay->m_bloom_filter_mutex); @@ -5257,7 +5208,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } } if (bad) { - Misbehaving(*peer, 100, "bad filteradd message"); + Misbehaving(*peer, "bad filteradd message"); } return; } diff --git a/src/net_processing.h b/src/net_processing.h index 85e399d948..bf9698ee02 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -16,6 +16,10 @@ class CChainParams; class CTxMemPool; class ChainstateManager; +namespace node { +class Warnings; +} // namespace node + /** Whether transaction reconciliation protocol should be enabled by default. */ static constexpr bool DEFAULT_TXRECONCILIATION_ENABLE{false}; /** Default for -maxorphantx, maximum number of orphan transactions kept in memory */ @@ -25,8 +29,6 @@ static const uint32_t DEFAULT_MAX_ORPHAN_TRANSACTIONS{100}; static const uint32_t DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN{100}; static const bool DEFAULT_PEERBLOOMFILTERS = false; static const bool DEFAULT_PEERBLOCKFILTERS = false; -/** Threshold for marking a node to be discouraged, e.g. disconnected and added to the discouragement filter. */ -static const int DISCOURAGEMENT_THRESHOLD{100}; /** Maximum number of outstanding CMPCTBLOCK requests for the same block. */ static const unsigned int MAX_CMPCTBLOCKS_INFLIGHT_PER_BLOCK = 3; @@ -73,7 +75,7 @@ public: static std::unique_ptr<PeerManager> make(CConnman& connman, AddrMan& addrman, BanMan* banman, ChainstateManager& chainman, - CTxMemPool& pool, Options opts); + CTxMemPool& pool, node::Warnings& warnings, Options opts); virtual ~PeerManager() { } /** @@ -104,7 +106,7 @@ public: virtual void SetBestBlock(int height, std::chrono::seconds time) = 0; /* Public for unit testing. */ - virtual void UnitTestMisbehaving(NodeId peer_id, int howmuch) = 0; + virtual void UnitTestMisbehaving(NodeId peer_id) = 0; /** * Evict extra outbound peers. If we think our tip may be stale, connect to an extra outbound. diff --git a/src/net_types.cpp b/src/net_types.cpp index fd6ad80404..fb6ad59599 100644 --- a/src/net_types.cpp +++ b/src/net_types.cpp @@ -43,7 +43,7 @@ UniValue BanMapToJson(const banmap_t& bans) const auto& ban_entry = it.second; UniValue j = ban_entry.ToJson(); j.pushKV(BANMAN_JSON_ADDR_KEY, address.ToString()); - bans_json.push_back(j); + bans_json.push_back(std::move(j)); } return bans_json; } diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 74ab6dd8d8..0053464822 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -20,6 +20,9 @@ #include <iterator> #include <tuple> +using util::ContainsNoNUL; +using util::HasPrefix; + CNetAddr::BIP155Network CNetAddr::GetBIP155Network() const { switch (m_net) { diff --git a/src/netaddress.h b/src/netaddress.h index ea2d14336e..52fecada1c 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -448,7 +448,7 @@ private: // Recognize NET_INTERNAL embedded in IPv6, such addresses are not // gossiped but could be coming from addrman, when unserializing from // disk. - if (HasPrefix(m_addr, INTERNAL_IN_IPV6_PREFIX)) { + if (util::HasPrefix(m_addr, INTERNAL_IN_IPV6_PREFIX)) { m_net = NET_INTERNAL; memmove(m_addr.data(), m_addr.data() + INTERNAL_IN_IPV6_PREFIX.size(), ADDR_INTERNAL_SIZE); @@ -456,8 +456,8 @@ private: return; } - if (!HasPrefix(m_addr, IPV4_IN_IPV6_PREFIX) && - !HasPrefix(m_addr, TORV2_IN_IPV6_PREFIX)) { + if (!util::HasPrefix(m_addr, IPV4_IN_IPV6_PREFIX) && + !util::HasPrefix(m_addr, TORV2_IN_IPV6_PREFIX)) { return; } diff --git a/src/netbase.cpp b/src/netbase.cpp index e231766487..f5f0997ba6 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -23,10 +23,12 @@ #include <limits> #include <memory> -#if HAVE_SOCKADDR_UN +#ifdef HAVE_SOCKADDR_UN #include <sys/un.h> #endif +using util::ContainsNoNUL; + // Settings static GlobalMutex g_proxyinfo_mutex; static Proxy proxyInfo[NET_MAX] GUARDED_BY(g_proxyinfo_mutex); @@ -216,7 +218,7 @@ CService LookupNumeric(const std::string& name, uint16_t portDefault, DNSLookupF bool IsUnixSocketPath(const std::string& name) { -#if HAVE_SOCKADDR_UN +#ifdef HAVE_SOCKADDR_UN if (name.find(ADDR_PREFIX_UNIX) != 0) return false; // Split off "unix:" prefix @@ -485,24 +487,23 @@ bool Socks5(const std::string& strDest, uint16_t port, const ProxyCredentials* a } } -std::unique_ptr<Sock> CreateSockOS(sa_family_t address_family) +std::unique_ptr<Sock> CreateSockOS(int domain, int type, int protocol) { // Not IPv4, IPv6 or UNIX - if (address_family == AF_UNSPEC) return nullptr; - - int protocol{IPPROTO_TCP}; -#if HAVE_SOCKADDR_UN - if (address_family == AF_UNIX) protocol = 0; -#endif + if (domain == AF_UNSPEC) return nullptr; // Create a socket in the specified address family. - SOCKET hSocket = socket(address_family, SOCK_STREAM, protocol); + SOCKET hSocket = socket(domain, type, protocol); if (hSocket == INVALID_SOCKET) { return nullptr; } auto sock = std::make_unique<Sock>(hSocket); + if (domain != AF_INET && domain != AF_INET6 && domain != AF_UNIX) { + return sock; + } + // Ensure that waiting for I/O on this socket won't result in undefined // behavior. if (!sock->IsSelectable()) { @@ -526,19 +527,22 @@ std::unique_ptr<Sock> CreateSockOS(sa_family_t address_family) return nullptr; } -#if HAVE_SOCKADDR_UN - if (address_family == AF_UNIX) return sock; +#ifdef HAVE_SOCKADDR_UN + if (domain == AF_UNIX) return sock; #endif - // Set the no-delay option (disable Nagle's algorithm) on the TCP socket. - const int on{1}; - if (sock->SetSockOpt(IPPROTO_TCP, TCP_NODELAY, &on, sizeof(on)) == SOCKET_ERROR) { - LogPrint(BCLog::NET, "Unable to set TCP_NODELAY on a newly created socket, continuing anyway\n"); + if (protocol == IPPROTO_TCP) { + // Set the no-delay option (disable Nagle's algorithm) on the TCP socket. + const int on{1}; + if (sock->SetSockOpt(IPPROTO_TCP, TCP_NODELAY, &on, sizeof(on)) == SOCKET_ERROR) { + LogPrint(BCLog::NET, "Unable to set TCP_NODELAY on a newly created socket, continuing anyway\n"); + } } + return sock; } -std::function<std::unique_ptr<Sock>(const sa_family_t&)> CreateSock = CreateSockOS; +std::function<std::unique_ptr<Sock>(int, int, int)> CreateSock = CreateSockOS; template<typename... Args> static void LogConnectFailure(bool manual_connection, const char* fmt, const Args&... args) { @@ -607,7 +611,7 @@ static bool ConnectToSocket(const Sock& sock, struct sockaddr* sockaddr, socklen std::unique_ptr<Sock> ConnectDirectly(const CService& dest, bool manual_connection) { - auto sock = CreateSock(dest.GetSAFamily()); + auto sock = CreateSock(dest.GetSAFamily(), SOCK_STREAM, IPPROTO_TCP); if (!sock) { LogPrintLevel(BCLog::NET, BCLog::Level::Error, "Cannot create a socket for connecting to %s\n", dest.ToStringAddrPort()); return {}; @@ -634,8 +638,8 @@ std::unique_ptr<Sock> Proxy::Connect() const if (!m_is_unix_socket) return ConnectDirectly(proxy, /*manual_connection=*/true); -#if HAVE_SOCKADDR_UN - auto sock = CreateSock(AF_UNIX); +#ifdef HAVE_SOCKADDR_UN + auto sock = CreateSock(AF_UNIX, SOCK_STREAM, 0); if (!sock) { LogPrintLevel(BCLog::NET, BCLog::Level::Error, "Cannot create a socket for connecting to %s\n", m_unix_socket_path); return {}; diff --git a/src/netbase.h b/src/netbase.h index 321c288f67..8ef6c28996 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -262,16 +262,18 @@ CService LookupNumeric(const std::string& name, uint16_t portDefault = 0, DNSLoo CSubNet LookupSubNet(const std::string& subnet_str); /** - * Create a TCP or UNIX socket in the given address family. - * @param[in] address_family to use for the socket. + * Create a real socket from the operating system. + * @param[in] domain Communications domain, first argument to the socket(2) syscall. + * @param[in] type Type of the socket, second argument to the socket(2) syscall. + * @param[in] protocol The particular protocol to be used with the socket, third argument to the socket(2) syscall. * @return pointer to the created Sock object or unique_ptr that owns nothing in case of failure */ -std::unique_ptr<Sock> CreateSockOS(sa_family_t address_family); +std::unique_ptr<Sock> CreateSockOS(int domain, int type, int protocol); /** * Socket factory. Defaults to `CreateSockOS()`, but can be overridden by unit tests. */ -extern std::function<std::unique_ptr<Sock>(const sa_family_t&)> CreateSock; +extern std::function<std::unique_ptr<Sock>(int, int, int)> CreateSock; /** * Create a socket and try to connect to the specified service. diff --git a/src/node/abort.cpp b/src/node/abort.cpp index b727608384..8a17c41fd2 100644 --- a/src/node/abort.cpp +++ b/src/node/abort.cpp @@ -6,19 +6,18 @@ #include <logging.h> #include <node/interface_ui.h> +#include <node/warnings.h> #include <util/signalinterrupt.h> #include <util/translation.h> -#include <warnings.h> #include <atomic> #include <cstdlib> -#include <string> namespace node { -void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const bilingual_str& message) +void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const bilingual_str& message, node::Warnings* warnings) { - SetMiscWarning(message); + if (warnings) warnings->Set(Warning::FATAL_INTERNAL_ERROR, message); InitError(_("A fatal internal error occurred, see debug.log for details: ") + message); exit_status.store(EXIT_FAILURE); if (shutdown && !(*shutdown)()) { diff --git a/src/node/abort.h b/src/node/abort.h index 1092279142..c881af4634 100644 --- a/src/node/abort.h +++ b/src/node/abort.h @@ -14,7 +14,8 @@ class SignalInterrupt; } // namespace util namespace node { -void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const bilingual_str& message); +class Warnings; +void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const bilingual_str& message, node::Warnings* warnings); } // namespace node #endif // BITCOIN_NODE_ABORT_H diff --git a/src/node/blockmanager_args.cpp b/src/node/blockmanager_args.cpp index dd8419a68a..fa76566652 100644 --- a/src/node/blockmanager_args.cpp +++ b/src/node/blockmanager_args.cpp @@ -33,8 +33,6 @@ util::Result<void> ApplyArgsManOptions(const ArgsManager& args, BlockManager::Op if (auto value{args.GetBoolArg("-fastprune")}) opts.fast_prune = *value; - if (auto value{args.GetBoolArg("-reindex")}) opts.reindex = *value; - return {}; } } // namespace node diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 4067ccee51..fb62e78138 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -551,7 +551,7 @@ bool BlockManager::LoadBlockIndexDB(const std::optional<uint256>& snapshot_block // Check whether we need to continue reindexing bool fReindexing = false; m_block_tree_db->ReadReindexing(fReindexing); - if (fReindexing) m_reindexing = true; + if (fReindexing) m_blockfiles_indexed = false; return true; } @@ -1182,7 +1182,7 @@ void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFile ImportingNow imp{chainman.m_blockman.m_importing}; // -reindex - if (chainman.m_blockman.m_reindexing) { + if (!chainman.m_blockman.m_blockfiles_indexed) { int nFile = 0; // Map of disk positions for blocks with unknown parent (only used for reindex); // parent hash -> child disk position, multiple children can have the same parent. @@ -1205,7 +1205,7 @@ void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFile nFile++; } WITH_LOCK(::cs_main, chainman.m_blockman.m_block_tree_db->WriteReindexing(false)); - chainman.m_blockman.m_reindexing = false; + chainman.m_blockman.m_blockfiles_indexed = true; LogPrintf("Reindexing finished\n"); // To avoid ending up in a situation without genesis block, re-try initializing (no-op if reindexing worked): chainman.ActiveChainstate().LoadGenesisBlock(); diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index a501067091..108a08a72b 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -267,18 +267,18 @@ public: explicit BlockManager(const util::SignalInterrupt& interrupt, Options opts) : m_prune_mode{opts.prune_target > 0}, m_opts{std::move(opts)}, - m_interrupt{interrupt}, - m_reindexing{m_opts.reindex} {}; + m_interrupt{interrupt} {} const util::SignalInterrupt& m_interrupt; std::atomic<bool> m_importing{false}; /** - * Tracks if a reindex is currently in progress. Set to true when a reindex - * is requested and false when reindexing completes. Its value is persisted - * in the BlockTreeDB across restarts. + * Whether all blockfiles have been added to the block tree database. + * Normally true, but set to false when a reindex is requested and the + * database is wiped. The value is persisted in the database across restarts + * and will be false until reindexing completes. */ - std::atomic_bool m_reindexing; + std::atomic_bool m_blockfiles_indexed{true}; BlockMap m_block_index GUARDED_BY(cs_main); @@ -359,7 +359,7 @@ public: [[nodiscard]] uint64_t GetPruneTarget() const { return m_opts.prune_target; } static constexpr auto PRUNE_TARGET_MANUAL{std::numeric_limits<uint64_t>::max()}; - [[nodiscard]] bool LoadingBlocks() const { return m_importing || m_reindexing; } + [[nodiscard]] bool LoadingBlocks() const { return m_importing || !m_blockfiles_indexed; } /** Calculate the amount of disk space the block & undo files currently use */ uint64_t CalculateCurrentUsage(); diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index d6eb14f513..d7e6176be1 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -45,11 +45,12 @@ static ChainstateLoadResult CompleteChainstateInitialization( .path = chainman.m_options.datadir / "blocks" / "index", .cache_bytes = static_cast<size_t>(cache_sizes.block_tree_db), .memory_only = options.block_tree_db_in_memory, - .wipe_data = options.reindex, + .wipe_data = options.wipe_block_tree_db, .options = chainman.m_options.block_tree_db}); - if (options.reindex) { + if (options.wipe_block_tree_db) { pblocktree->WriteReindexing(true); + chainman.m_blockman.m_blockfiles_indexed = false; //If we're reindexing in prune mode, wipe away unusable block files and all undo data files if (options.prune) { chainman.m_blockman.CleanupBlockRevFiles(); @@ -60,8 +61,7 @@ static ChainstateLoadResult CompleteChainstateInitialization( // LoadBlockIndex will load m_have_pruned if we've ever removed a // block file from disk. - // Note that it also sets m_reindexing based on the disk flag! - // From here on, m_reindexing and options.reindex values may be different! + // Note that it also sets m_blockfiles_indexed based on the disk flag! if (!chainman.LoadBlockIndex()) { if (chainman.m_interrupt) return {ChainstateLoadStatus::INTERRUPTED, {}}; return {ChainstateLoadStatus::FAILURE, _("Error loading block database")}; @@ -84,12 +84,12 @@ static ChainstateLoadResult CompleteChainstateInitialization( // If we're not mid-reindex (based on disk + args), add a genesis block on disk // (otherwise we use the one already on disk). // This is called again in ImportBlocks after the reindex completes. - if (!chainman.m_blockman.m_reindexing && !chainman.ActiveChainstate().LoadGenesisBlock()) { + if (chainman.m_blockman.m_blockfiles_indexed && !chainman.ActiveChainstate().LoadGenesisBlock()) { return {ChainstateLoadStatus::FAILURE, _("Error initializing block database")}; } auto is_coinsview_empty = [&](Chainstate* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { - return options.reindex || options.reindex_chainstate || chainstate->CoinsTip().GetBestBlock().IsNull(); + return options.wipe_chainstate_db || chainstate->CoinsTip().GetBestBlock().IsNull(); }; assert(chainman.m_total_coinstip_cache > 0); @@ -110,7 +110,7 @@ static ChainstateLoadResult CompleteChainstateInitialization( chainstate->InitCoinsDB( /*cache_size_bytes=*/chainman.m_total_coinsdb_cache * init_cache_fraction, /*in_memory=*/options.coins_db_in_memory, - /*should_wipe=*/options.reindex || options.reindex_chainstate); + /*should_wipe=*/options.wipe_chainstate_db); if (options.coins_error_cb) { chainstate->CoinsErrorCatcher().AddReadErrCallback(options.coins_error_cb); @@ -142,7 +142,7 @@ static ChainstateLoadResult CompleteChainstateInitialization( } } - if (!options.reindex) { + if (!options.wipe_block_tree_db) { auto chainstates{chainman.GetAll()}; if (std::any_of(chainstates.begin(), chainstates.end(), [](const Chainstate* cs) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return cs->NeedsRedownload(); })) { @@ -188,7 +188,7 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize // Load a chain created from a UTXO snapshot, if any exist. bool has_snapshot = chainman.DetectSnapshotChainstate(); - if (has_snapshot && (options.reindex || options.reindex_chainstate)) { + if (has_snapshot && options.wipe_chainstate_db) { LogPrintf("[snapshot] deleting snapshot chainstate due to reindexing\n"); if (!chainman.DeleteSnapshotChainstate()) { return {ChainstateLoadStatus::FAILURE_FATAL, Untranslated("Couldn't remove snapshot chainstate.")}; @@ -247,7 +247,7 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize ChainstateLoadResult VerifyLoadedChainstate(ChainstateManager& chainman, const ChainstateLoadOptions& options) { auto is_coinsview_empty = [&](Chainstate* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { - return options.reindex || options.reindex_chainstate || chainstate->CoinsTip().GetBestBlock().IsNull(); + return options.wipe_chainstate_db || chainstate->CoinsTip().GetBestBlock().IsNull(); }; LOCK(cs_main); diff --git a/src/node/chainstate.h b/src/node/chainstate.h index a6e9a0331b..bb0c4f2b87 100644 --- a/src/node/chainstate.h +++ b/src/node/chainstate.h @@ -22,8 +22,13 @@ struct ChainstateLoadOptions { CTxMemPool* mempool{nullptr}; bool block_tree_db_in_memory{false}; bool coins_db_in_memory{false}; - bool reindex{false}; - bool reindex_chainstate{false}; + // Whether to wipe the block tree database when loading it. If set, this + // will also set a reindexing flag so any existing block data files will be + // scanned and added to the database. + bool wipe_block_tree_db{false}; + // Whether to wipe the chainstate database when loading it. If set, this + // will cause the chainstate database to be rebuilt starting from genesis. + bool wipe_chainstate_db{false}; bool prune{false}; //! Setting require_full_verification to true will require all checks at //! check_level (below) to succeed for loading to succeed. Setting it to diff --git a/src/node/chainstatemanager_args.cpp b/src/node/chainstatemanager_args.cpp index 1cc126cb05..bc4a815a3e 100644 --- a/src/node/chainstatemanager_args.cpp +++ b/src/node/chainstatemanager_args.cpp @@ -24,7 +24,10 @@ namespace node { util::Result<void> ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts) { - if (auto value{args.GetBoolArg("-checkblockindex")}) opts.check_block_index = *value; + if (auto value{args.GetIntArg("-checkblockindex")}) { + // Interpret bare -checkblockindex argument as 1 instead of 0. + opts.check_block_index = args.GetArg("-checkblockindex")->empty() ? 1 : *value; + } if (auto value{args.GetBoolArg("-checkpoints")}) opts.checkpoints_enabled = *value; diff --git a/src/node/context.cpp b/src/node/context.cpp index e32d21b383..75dfaee866 100644 --- a/src/node/context.cpp +++ b/src/node/context.cpp @@ -7,12 +7,14 @@ #include <addrman.h> #include <banman.h> #include <interfaces/chain.h> +#include <interfaces/mining.h> #include <kernel/context.h> #include <key.h> #include <net.h> #include <net_processing.h> #include <netgroup.h> #include <node/kernel_notifications.h> +#include <node/warnings.h> #include <policy/fees.h> #include <scheduler.h> #include <txmempool.h> diff --git a/src/node/context.h b/src/node/context.h index a7d92989dd..a664fad80b 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -27,6 +27,7 @@ class PeerManager; namespace interfaces { class Chain; class ChainClient; +class Mining; class Init; class WalletLoader; } // namespace interfaces @@ -39,6 +40,7 @@ class SignalInterrupt; namespace node { class KernelNotifications; +class Warnings; //! NodeContext struct containing references to chain state and connection //! state. @@ -73,6 +75,7 @@ struct NodeContext { std::vector<std::unique_ptr<interfaces::ChainClient>> chain_clients; //! Reference to chain client that should used to load or create wallets //! opened by the gui. + std::unique_ptr<interfaces::Mining> mining; interfaces::WalletLoader* wallet_loader{nullptr}; std::unique_ptr<CScheduler> scheduler; std::function<void()> rpc_interruption_point = [] {}; @@ -81,6 +84,8 @@ struct NodeContext { //! Issues calls about blocks and transactions std::unique_ptr<ValidationSignals> validation_signals; std::atomic<int> exit_status{EXIT_SUCCESS}; + //! Manages all the node warnings + std::unique_ptr<node::Warnings> warnings; //! Declare default constructor and destructor that are not inline, so code //! instantiating the NodeContext struct doesn't need to #include class diff --git a/src/node/interface_ui.cpp b/src/node/interface_ui.cpp index 9dd1e7d9cf..4f4d240d1b 100644 --- a/src/node/interface_ui.cpp +++ b/src/node/interface_ui.cpp @@ -10,6 +10,8 @@ #include <boost/signals2/optional_last_value.hpp> #include <boost/signals2/signal.hpp> +using util::MakeUnorderedList; + CClientUIInterface uiInterface; struct UISignals { diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 216f44ab9e..e0bab6e22e 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -8,12 +8,14 @@ #include <chain.h> #include <chainparams.h> #include <common/args.h> +#include <consensus/validation.h> #include <deploymentstatus.h> #include <external_signer.h> #include <index/blockfilterindex.h> #include <init.h> #include <interfaces/chain.h> #include <interfaces/handler.h> +#include <interfaces/mining.h> #include <interfaces/node.h> #include <interfaces/wallet.h> #include <kernel/chain.h> @@ -30,7 +32,10 @@ #include <node/context.h> #include <node/interface_ui.h> #include <node/mini_miner.h> +#include <node/miner.h> #include <node/transaction.h> +#include <node/types.h> +#include <node/warnings.h> #include <policy/feerate.h> #include <policy/fees.h> #include <policy/policy.h> @@ -52,7 +57,6 @@ #include <util/translation.h> #include <validation.h> #include <validationinterface.h> -#include <warnings.h> #include <config/bitcoin-config.h> // IWYU pragma: keep @@ -68,8 +72,11 @@ using interfaces::Chain; using interfaces::FoundBlock; using interfaces::Handler; using interfaces::MakeSignalHandler; +using interfaces::Mining; using interfaces::Node; using interfaces::WalletLoader; +using node::BlockAssembler; +using util::Join; namespace node { // All members of the classes in this namespace are intentionally public, as the @@ -91,7 +98,7 @@ public: explicit NodeImpl(NodeContext& context) { setContext(&context); } void initLogging() override { InitLogging(args()); } void initParameterInteraction() override { InitParameterInteraction(args()); } - bilingual_str getWarnings() override { return Join(GetWarnings(), Untranslated("<hr />")); } + bilingual_str getWarnings() override { return Join(Assert(m_context->warnings)->GetMessages(), Untranslated("<hr />")); } int getExitStatus() override { return Assert(m_context)->exit_status.load(); } uint32_t getLogCategories() override { return LogInstance().GetCategoryMask(); } bool baseInitialize() override @@ -99,6 +106,7 @@ public: if (!AppInitBasicSetup(args(), Assert(context())->exit_status)) return false; if (!AppInitParameterInteraction(args())) return false; + m_context->warnings = std::make_unique<node::Warnings>(); m_context->kernel = std::make_unique<kernel::Context>(); m_context->ecc_context = std::make_unique<ECC_Context>(); if (!AppInitSanityChecks(*m_context->kernel)) return false; @@ -123,7 +131,7 @@ public: void startShutdown() override { if (!(*Assert(Assert(m_context)->shutdown))()) { - LogPrintf("Error: failed to send shutdown signal\n"); + LogError("Failed to send shutdown signal\n"); } // Stop RPC for clean shutdown if any of waitfor* commands is executed. if (args().GetBoolArg("-server", false)) { @@ -828,10 +836,64 @@ public: ValidationSignals& validation_signals() { return *Assert(m_node.validation_signals); } NodeContext& m_node; }; + +class MinerImpl : public Mining +{ +public: + explicit MinerImpl(NodeContext& node) : m_node(node) {} + + bool isTestChain() override + { + return chainman().GetParams().IsTestChain(); + } + + bool isInitialBlockDownload() override + { + return chainman().IsInitialBlockDownload(); + } + + std::optional<uint256> getTipHash() override + { + LOCK(::cs_main); + CBlockIndex* tip{chainman().ActiveChain().Tip()}; + if (!tip) return {}; + return tip->GetBlockHash(); + } + + bool processNewBlock(const std::shared_ptr<const CBlock>& block, bool* new_block) override + { + return chainman().ProcessNewBlock(block, /*force_processing=*/true, /*min_pow_checked=*/true, /*new_block=*/new_block); + } + + unsigned int getTransactionsUpdated() override + { + return context()->mempool->GetTransactionsUpdated(); + } + + bool testBlockValidity(BlockValidationState& state, const CBlock& block, bool check_merkle_root) override + { + LOCK(::cs_main); + return TestBlockValidity(state, chainman().GetParams(), chainman().ActiveChainstate(), block, chainman().ActiveChain().Tip(), /*fCheckPOW=*/false, check_merkle_root); + } + + std::unique_ptr<CBlockTemplate> createNewBlock(const CScript& script_pub_key, bool use_mempool) override + { + BlockAssembler::Options options; + ApplyArgsManOptions(gArgs, options); + + LOCK(::cs_main); + return BlockAssembler{chainman().ActiveChainstate(), use_mempool ? context()->mempool.get() : nullptr, options}.CreateNewBlock(script_pub_key); + } + + NodeContext* context() override { return &m_node; } + ChainstateManager& chainman() { return *Assert(m_node.chainman); } + NodeContext& m_node; +}; } // namespace } // namespace node namespace interfaces { std::unique_ptr<Node> MakeNode(node::NodeContext& context) { return std::make_unique<node::NodeImpl>(context); } std::unique_ptr<Chain> MakeChain(node::NodeContext& context) { return std::make_unique<node::ChainImpl>(context); } +std::unique_ptr<Mining> MakeMining(node::NodeContext& context) { return std::make_unique<node::MinerImpl>(context); } } // namespace interfaces diff --git a/src/node/kernel_notifications.cpp b/src/node/kernel_notifications.cpp index e326d4a1f2..9894052a3a 100644 --- a/src/node/kernel_notifications.cpp +++ b/src/node/kernel_notifications.cpp @@ -10,23 +10,25 @@ #include <common/args.h> #include <common/system.h> #include <kernel/context.h> +#include <kernel/warning.h> #include <logging.h> #include <node/abort.h> #include <node/interface_ui.h> +#include <node/warnings.h> #include <util/check.h> #include <util/signalinterrupt.h> #include <util/strencodings.h> #include <util/string.h> #include <util/translation.h> -#include <warnings.h> #include <cstdint> #include <string> #include <thread> +using util::ReplaceAll; + static void AlertNotify(const std::string& strMessage) { - uiInterface.NotifyAlertChanged(); #if HAVE_SYSTEM std::string strCmd = gArgs.GetArg("-alertnotify", ""); if (strCmd.empty()) return; @@ -44,16 +46,6 @@ static void AlertNotify(const std::string& strMessage) #endif } -static void DoWarning(const bilingual_str& warning) -{ - static bool fWarned = false; - SetMiscWarning(warning); - if (!fWarned) { - AlertNotify(warning.original); - fWarned = true; - } -} - namespace node { kernel::InterruptResult KernelNotifications::blockTip(SynchronizationState state, CBlockIndex& index) @@ -61,7 +53,7 @@ kernel::InterruptResult KernelNotifications::blockTip(SynchronizationState state uiInterface.NotifyBlockTip(state, &index); if (m_stop_at_height && index.nHeight >= m_stop_at_height) { if (!m_shutdown()) { - LogPrintf("Error: failed to send shutdown signal after reaching stop height\n"); + LogError("Failed to send shutdown signal after reaching stop height\n"); } return kernel::Interrupted{}; } @@ -78,20 +70,27 @@ void KernelNotifications::progress(const bilingual_str& title, int progress_perc uiInterface.ShowProgress(title.translated, progress_percent, resume_possible); } -void KernelNotifications::warning(const bilingual_str& warning) +void KernelNotifications::warningSet(kernel::Warning id, const bilingual_str& message) +{ + if (m_warnings.Set(id, message)) { + AlertNotify(message.original); + } +} + +void KernelNotifications::warningUnset(kernel::Warning id) { - DoWarning(warning); + m_warnings.Unset(id); } void KernelNotifications::flushError(const bilingual_str& message) { - AbortNode(&m_shutdown, m_exit_status, message); + AbortNode(&m_shutdown, m_exit_status, message, &m_warnings); } void KernelNotifications::fatalError(const bilingual_str& message) { node::AbortNode(m_shutdown_on_fatal_error ? &m_shutdown : nullptr, - m_exit_status, message); + m_exit_status, message, &m_warnings); } void ReadNotificationArgs(const ArgsManager& args, KernelNotifications& notifications) diff --git a/src/node/kernel_notifications.h b/src/node/kernel_notifications.h index f4d97a0fff..e37f4d4e1e 100644 --- a/src/node/kernel_notifications.h +++ b/src/node/kernel_notifications.h @@ -15,18 +15,24 @@ class CBlockIndex; enum class SynchronizationState; struct bilingual_str; +namespace kernel { +enum class Warning; +} // namespace kernel + namespace util { class SignalInterrupt; } // namespace util namespace node { +class Warnings; static constexpr int DEFAULT_STOPATHEIGHT{0}; class KernelNotifications : public kernel::Notifications { public: - KernelNotifications(util::SignalInterrupt& shutdown, std::atomic<int>& exit_status) : m_shutdown(shutdown), m_exit_status{exit_status} {} + KernelNotifications(util::SignalInterrupt& shutdown, std::atomic<int>& exit_status, node::Warnings& warnings) + : m_shutdown(shutdown), m_exit_status{exit_status}, m_warnings{warnings} {} [[nodiscard]] kernel::InterruptResult blockTip(SynchronizationState state, CBlockIndex& index) override; @@ -34,7 +40,9 @@ public: void progress(const bilingual_str& title, int progress_percent, bool resume_possible) override; - void warning(const bilingual_str& warning) override; + void warningSet(kernel::Warning id, const bilingual_str& message) override; + + void warningUnset(kernel::Warning id) override; void flushError(const bilingual_str& message) override; @@ -47,6 +55,7 @@ public: private: util::SignalInterrupt& m_shutdown; std::atomic<int>& m_exit_status; + node::Warnings& m_warnings; }; void ReadNotificationArgs(const ArgsManager& args, KernelNotifications& notifications); diff --git a/src/node/mempool_args.cpp b/src/node/mempool_args.cpp index ac26600919..f329affb1d 100644 --- a/src/node/mempool_args.cpp +++ b/src/node/mempool_args.cpp @@ -8,19 +8,20 @@ #include <kernel/mempool_options.h> #include <common/args.h> +#include <common/messages.h> #include <consensus/amount.h> #include <kernel/chainparams.h> #include <logging.h> #include <policy/feerate.h> #include <policy/policy.h> #include <tinyformat.h> -#include <util/error.h> #include <util/moneystr.h> #include <util/translation.h> #include <chrono> #include <memory> +using common::AmountErrMsg; using kernel::MemPoolLimits; using kernel::MemPoolOptions; diff --git a/src/node/miner.cpp b/src/node/miner.cpp index 87f40e993f..03c6d74deb 100644 --- a/src/node/miner.cpp +++ b/src/node/miner.cpp @@ -80,15 +80,6 @@ void ApplyArgsManOptions(const ArgsManager& args, BlockAssembler::Options& optio if (const auto parsed{ParseMoney(*blockmintxfee)}) options.blockMinFeeRate = CFeeRate{*parsed}; } } -static BlockAssembler::Options ConfiguredOptions() -{ - BlockAssembler::Options options; - ApplyArgsManOptions(gArgs, options); - return options; -} - -BlockAssembler::BlockAssembler(Chainstate& chainstate, const CTxMemPool* mempool) - : BlockAssembler(chainstate, mempool, ConfiguredOptions()) {} void BlockAssembler::resetBlock() { diff --git a/src/node/miner.h b/src/node/miner.h index 06a917228d..c3178a7532 100644 --- a/src/node/miner.h +++ b/src/node/miner.h @@ -161,7 +161,6 @@ public: bool test_block_validity{true}; }; - explicit BlockAssembler(Chainstate& chainstate, const CTxMemPool* mempool); explicit BlockAssembler(Chainstate& chainstate, const CTxMemPool* mempool, const Options& options); /** Construct a new block template with coinbase to scriptPubKeyIn */ diff --git a/src/node/timeoffsets.cpp b/src/node/timeoffsets.cpp index 62f527be8a..002c00d245 100644 --- a/src/node/timeoffsets.cpp +++ b/src/node/timeoffsets.cpp @@ -3,13 +3,12 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <logging.h> -#include <node/interface_ui.h> #include <node/timeoffsets.h> +#include <node/warnings.h> #include <sync.h> #include <tinyformat.h> #include <util/time.h> #include <util/translation.h> -#include <warnings.h> #include <algorithm> #include <chrono> @@ -49,8 +48,7 @@ bool TimeOffsets::WarnIfOutOfSync() const // when median == std::numeric_limits<int64_t>::min(), calling std::chrono::abs is UB auto median{std::max(Median(), std::chrono::seconds(std::numeric_limits<int64_t>::min() + 1))}; if (std::chrono::abs(median) <= WARN_THRESHOLD) { - SetMedianTimeOffsetWarning(std::nullopt); - uiInterface.NotifyAlertChanged(); + m_warnings.Unset(node::Warning::CLOCK_OUT_OF_SYNC); return false; } @@ -63,7 +61,6 @@ bool TimeOffsets::WarnIfOutOfSync() const "RPC methods to get more info." ), Ticks<std::chrono::minutes>(WARN_THRESHOLD))}; LogWarning("%s\n", msg.original); - SetMedianTimeOffsetWarning(msg); - uiInterface.NotifyAlertChanged(); + m_warnings.Set(node::Warning::CLOCK_OUT_OF_SYNC, msg); return true; } diff --git a/src/node/timeoffsets.h b/src/node/timeoffsets.h index 2b12584e12..eba706ac1e 100644 --- a/src/node/timeoffsets.h +++ b/src/node/timeoffsets.h @@ -11,8 +11,16 @@ #include <cstddef> #include <deque> +namespace node { +class Warnings; +} // namespace node + class TimeOffsets { +public: + TimeOffsets(node::Warnings& warnings) : m_warnings{warnings} {} + +private: //! Maximum number of timeoffsets stored. static constexpr size_t MAX_SIZE{50}; //! Minimum difference between system and network time for a warning to be raised. @@ -23,6 +31,8 @@ class TimeOffsets * positive offset means our peer's clock is ahead of our local clock. */ std::deque<std::chrono::seconds> m_offsets GUARDED_BY(m_mutex){}; + node::Warnings& m_warnings; + public: /** Add a new time offset sample. */ void Add(std::chrono::seconds offset) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index b66a4f2f39..591dcd698d 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -9,6 +9,7 @@ #include <net_processing.h> #include <node/blockstorage.h> #include <node/context.h> +#include <node/types.h> #include <txmempool.h> #include <validation.h> #include <validationinterface.h> diff --git a/src/node/transaction.h b/src/node/transaction.h index 6782536ace..5f524f4e28 100644 --- a/src/node/transaction.h +++ b/src/node/transaction.h @@ -5,9 +5,9 @@ #ifndef BITCOIN_NODE_TRANSACTION_H #define BITCOIN_NODE_TRANSACTION_H +#include <common/messages.h> #include <policy/feerate.h> #include <primitives/transaction.h> -#include <util/error.h> class CBlockIndex; class CTxMemPool; diff --git a/src/node/types.h b/src/node/types.h new file mode 100644 index 0000000000..0461e85f43 --- /dev/null +++ b/src/node/types.h @@ -0,0 +1,29 @@ +// Copyright (c) 2010-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +//! @file node/types.h is a home for public enum and struct type definitions +//! that are used by internally by node code, but also used externally by wallet +//! or GUI code. +//! +//! This file is intended to define only simple types that do not have external +//! dependencies. More complicated types should be defined in dedicated header +//! files. + +#ifndef BITCOIN_NODE_TYPES_H +#define BITCOIN_NODE_TYPES_H + +namespace node { +enum class TransactionError { + OK, //!< No error + MISSING_INPUTS, + ALREADY_IN_CHAIN, + MEMPOOL_REJECTED, + MEMPOOL_ERROR, + MAX_FEE_EXCEEDED, + MAX_BURN_EXCEEDED, + INVALID_PACKAGE, +}; +} // namespace node + +#endif // BITCOIN_NODE_TYPES_H diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h index 1160bb55f0..a7c4135787 100644 --- a/src/node/utxo_snapshot.h +++ b/src/node/utxo_snapshot.h @@ -6,16 +6,23 @@ #ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H #define BITCOIN_NODE_UTXO_SNAPSHOT_H +#include <chainparams.h> +#include <kernel/chainparams.h> #include <kernel/cs_main.h> #include <serialize.h> #include <sync.h> #include <uint256.h> +#include <util/chaintype.h> +#include <util/check.h> #include <util/fs.h> #include <cstdint> #include <optional> #include <string_view> +// UTXO set snapshot magic bytes +static constexpr std::array<uint8_t, 5> SNAPSHOT_MAGIC_BYTES = {'u', 't', 'x', 'o', 0xff}; + class Chainstate; namespace node { @@ -23,23 +30,78 @@ namespace node { //! assumeutxo Chainstate can be constructed. class SnapshotMetadata { + const uint16_t m_version{1}; + const std::set<uint16_t> m_supported_versions{1}; + const MessageStartChars m_network_magic; public: //! The hash of the block that reflects the tip of the chain for the //! UTXO set contained in this snapshot. uint256 m_base_blockhash; + uint32_t m_base_blockheight; + //! The number of coins in the UTXO set contained in this snapshot. Used //! during snapshot load to estimate progress of UTXO set reconstruction. uint64_t m_coins_count = 0; - SnapshotMetadata() { } SnapshotMetadata( + const MessageStartChars network_magic) : + m_network_magic(network_magic) { } + SnapshotMetadata( + const MessageStartChars network_magic, const uint256& base_blockhash, + const int base_blockheight, uint64_t coins_count) : + m_network_magic(network_magic), m_base_blockhash(base_blockhash), + m_base_blockheight(base_blockheight), m_coins_count(coins_count) { } - SERIALIZE_METHODS(SnapshotMetadata, obj) { READWRITE(obj.m_base_blockhash, obj.m_coins_count); } + template <typename Stream> + inline void Serialize(Stream& s) const { + s << SNAPSHOT_MAGIC_BYTES; + s << m_version; + s << m_network_magic; + s << m_base_blockheight; + s << m_base_blockhash; + s << m_coins_count; + } + + template <typename Stream> + inline void Unserialize(Stream& s) { + // Read the snapshot magic bytes + std::array<uint8_t, SNAPSHOT_MAGIC_BYTES.size()> snapshot_magic; + s >> snapshot_magic; + if (snapshot_magic != SNAPSHOT_MAGIC_BYTES) { + throw std::ios_base::failure("Invalid UTXO set snapshot magic bytes. Please check if this is indeed a snapshot file or if you are using an outdated snapshot format."); + } + + // Read the version + uint16_t version; + s >> version; + if (m_supported_versions.find(version) == m_supported_versions.end()) { + throw std::ios_base::failure(strprintf("Version of snapshot %s does not match any of the supported versions.", version)); + } + + // Read the network magic (pchMessageStart) + MessageStartChars message; + s >> message; + if (!std::equal(message.begin(), message.end(), m_network_magic.data())) { + auto metadata_network{GetNetworkForMagic(message)}; + if (metadata_network) { + std::string network_string{ChainTypeToString(metadata_network.value())}; + auto node_network{GetNetworkForMagic(m_network_magic)}; + std::string node_network_string{ChainTypeToString(node_network.value())}; + throw std::ios_base::failure(strprintf("The network of the snapshot (%s) does not match the network of this node (%s).", network_string, node_network_string)); + } else { + throw std::ios_base::failure("This snapshot has been created for an unrecognized network. This could be a custom signet, a new testnet or possibly caused by data corruption."); + } + } + + s >> m_base_blockheight; + s >> m_base_blockhash; + s >> m_coins_count; + } }; //! The file in the snapshot chainstate dir which stores the base blockhash. This is diff --git a/src/node/warnings.cpp b/src/node/warnings.cpp new file mode 100644 index 0000000000..b99c845900 --- /dev/null +++ b/src/node/warnings.cpp @@ -0,0 +1,68 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <config/bitcoin-config.h> // IWYU pragma: keep + +#include <node/warnings.h> + +#include <common/system.h> +#include <node/interface_ui.h> +#include <sync.h> +#include <univalue.h> +#include <util/translation.h> + +#include <utility> +#include <vector> + +namespace node { +Warnings::Warnings() +{ + // Pre-release build warning + if (!CLIENT_VERSION_IS_RELEASE) { + m_warnings.insert( + {Warning::PRE_RELEASE_TEST_BUILD, + _("This is a pre-release test build - use at your own risk - do not use for mining or merchant applications")}); + } +} +bool Warnings::Set(warning_type id, bilingual_str message) +{ + LOCK(m_mutex); + const auto& [_, inserted]{m_warnings.insert({id, std::move(message)})}; + if (inserted) uiInterface.NotifyAlertChanged(); + return inserted; +} + +bool Warnings::Unset(warning_type id) +{ + auto success{WITH_LOCK(m_mutex, return m_warnings.erase(id))}; + if (success) uiInterface.NotifyAlertChanged(); + return success; +} + +std::vector<bilingual_str> Warnings::GetMessages() const +{ + LOCK(m_mutex); + std::vector<bilingual_str> messages; + messages.reserve(m_warnings.size()); + for (const auto& [id, msg] : m_warnings) { + messages.push_back(msg); + } + return messages; +} + +UniValue GetWarningsForRpc(const Warnings& warnings, bool use_deprecated) +{ + if (use_deprecated) { + const auto all_messages{warnings.GetMessages()}; + return all_messages.empty() ? "" : all_messages.back().original; + } + + UniValue messages{UniValue::VARR}; + for (auto&& message : warnings.GetMessages()) { + messages.push_back(std::move(message.original)); + } + return messages; +} +} // namespace node diff --git a/src/node/warnings.h b/src/node/warnings.h new file mode 100644 index 0000000000..24aeb8a922 --- /dev/null +++ b/src/node/warnings.h @@ -0,0 +1,90 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_WARNINGS_H +#define BITCOIN_NODE_WARNINGS_H + +#include <sync.h> +#include <util/translation.h> + +#include <map> +#include <variant> +#include <vector> + +class UniValue; + +namespace kernel { +enum class Warning; +} // namespace kernel + +namespace node { +enum class Warning { + CLOCK_OUT_OF_SYNC, + PRE_RELEASE_TEST_BUILD, + FATAL_INTERNAL_ERROR, +}; + +/** + * @class Warnings + * @brief Manages warning messages within a node. + * + * The Warnings class provides mechanisms to set, unset, and retrieve + * warning messages. It updates the GUI when warnings are changed. + * + * This class is designed to be non-copyable to ensure warnings + * are managed centrally. + */ +class Warnings +{ + typedef std::variant<kernel::Warning, node::Warning> warning_type; + + mutable Mutex m_mutex; + std::map<warning_type, bilingual_str> m_warnings GUARDED_BY(m_mutex); + +public: + Warnings(); + //! A warnings instance should always be passed by reference, never copied. + Warnings(const Warnings&) = delete; + Warnings& operator=(const Warnings&) = delete; + /** + * @brief Set a warning message. If a warning with the specified + * `id` is already active, false is returned and the new + * warning is ignored. If `id` does not yet exist, the + * warning is set, the UI is updated, and true is returned. + * + * @param[in] id Unique identifier of the warning. + * @param[in] message Warning message to be shown. + * + * @returns true if the warning was indeed set (i.e. there is no + * active warning with this `id`), otherwise false. + */ + bool Set(warning_type id, bilingual_str message) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); + /** + * @brief Unset a warning message. If a warning with the specified + * `id` is active, it is unset, the UI is updated, and true + * is returned. Otherwise, no warning is unset and false is + * returned. + * + * @param[in] id Unique identifier of the warning. + * + * @returns true if the warning was indeed unset (i.e. there is an + * active warning with this `id`), otherwise false. + */ + bool Unset(warning_type id) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); + /** Return potential problems detected by the node, sorted by the + * warning_type id */ + std::vector<bilingual_str> GetMessages() const EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); +}; + +/** + * RPC helper function that wraps warnings.GetMessages(). + * + * Returns a UniValue::VSTR with the latest warning if use_deprecated is + * set to true, or a UniValue::VARR with all warnings otherwise. + */ +UniValue GetWarningsForRpc(const Warnings& warnings, bool use_deprecated); +} // namespace node + +#endif // BITCOIN_NODE_WARNINGS_H diff --git a/src/outputtype.cpp b/src/outputtype.cpp index c72d9deacb..8c2b76494b 100644 --- a/src/outputtype.cpp +++ b/src/outputtype.cpp @@ -81,11 +81,11 @@ std::vector<CTxDestination> GetAllDestinationsForKey(const CPubKey& key) } } -CTxDestination AddAndGetDestinationForScript(FillableSigningProvider& keystore, const CScript& script, OutputType type) +CTxDestination AddAndGetDestinationForScript(FlatSigningProvider& keystore, const CScript& script, OutputType type) { // Add script to keystore - keystore.AddCScript(script); - // Note that scripts over MAX_SCRIPT_ELEMENT_SIZE bytes are not yet supported. + keystore.scripts.emplace(CScriptID(script), script); + switch (type) { case OutputType::LEGACY: return ScriptHash(script); @@ -94,7 +94,7 @@ CTxDestination AddAndGetDestinationForScript(FillableSigningProvider& keystore, CTxDestination witdest = WitnessV0ScriptHash(script); CScript witprog = GetScriptForDestination(witdest); // Add the redeemscript, so that P2WSH and P2SH-P2WSH outputs are recognized as ours. - keystore.AddCScript(witprog); + keystore.scripts.emplace(CScriptID(witprog), witprog); if (type == OutputType::BECH32) { return witdest; } else { diff --git a/src/outputtype.h b/src/outputtype.h index a2d5942320..feef7991a6 100644 --- a/src/outputtype.h +++ b/src/outputtype.h @@ -46,7 +46,7 @@ std::vector<CTxDestination> GetAllDestinationsForKey(const CPubKey& key); * This function will automatically add the script (and any other * necessary scripts) to the keystore. */ -CTxDestination AddAndGetDestinationForScript(FillableSigningProvider& keystore, const CScript& script, OutputType); +CTxDestination AddAndGetDestinationForScript(FlatSigningProvider& keystore, const CScript& script, OutputType); /** Get the OutputType for a CTxDestination */ std::optional<OutputType> OutputTypeFromDestination(const CTxDestination& dest); diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index d8b4b907e4..e84a8428bf 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -93,7 +93,7 @@ bool IsStandard(const CScript& scriptPubKey, const std::optional<unsigned>& max_ bool IsStandardTx(const CTransaction& tx, const std::optional<unsigned>& max_datacarrier_bytes, bool permit_bare_multisig, const CFeeRate& dust_relay_fee, std::string& reason) { - if (tx.nVersion > TX_MAX_STANDARD_VERSION || tx.nVersion < 1) { + if (tx.version > TX_MAX_STANDARD_VERSION || tx.version < 1) { reason = "version"; return false; } diff --git a/src/policy/policy.h b/src/policy/policy.h index 6a7980c312..a82488a28c 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -131,7 +131,7 @@ bool IsStandard(const CScript& scriptPubKey, const std::optional<unsigned>& max_ // Changing the default transaction version requires a two step process: first // adapting relay policy by bumping TX_MAX_STANDARD_VERSION, and then later // allowing the new transaction version in the wallet/RPC. -static constexpr decltype(CTransaction::nVersion) TX_MAX_STANDARD_VERSION{2}; +static constexpr decltype(CTransaction::version) TX_MAX_STANDARD_VERSION{3}; /** * Check for standard transaction types diff --git a/src/policy/v3_policy.cpp b/src/policy/v3_policy.cpp index 3c3942d707..6bd043b8e3 100644 --- a/src/policy/v3_policy.cpp +++ b/src/policy/v3_policy.cpp @@ -43,13 +43,13 @@ struct ParentInfo { const Txid& m_txid; /** Wtxid used for debug string */ const Wtxid& m_wtxid; - /** nVersion used to check inheritance of v3 and non-v3 */ - decltype(CTransaction::nVersion) m_version; + /** version used to check inheritance of v3 and non-v3 */ + decltype(CTransaction::version) m_version; /** If parent is in mempool, whether it has any descendants in mempool. */ bool m_has_mempool_descendant; ParentInfo() = delete; - ParentInfo(const Txid& txid, const Wtxid& wtxid, decltype(CTransaction::nVersion) version, bool has_mempool_descendant) : + ParentInfo(const Txid& txid, const Wtxid& wtxid, decltype(CTransaction::version) version, bool has_mempool_descendant) : m_txid{txid}, m_wtxid{wtxid}, m_version{version}, m_has_mempool_descendant{has_mempool_descendant} {} @@ -66,7 +66,13 @@ std::optional<std::string> PackageV3Checks(const CTransactionRef& ptx, int64_t v const auto in_package_parents{FindInPackageParents(package, ptx)}; // Now we have all ancestors, so we can start checking v3 rules. - if (ptx->nVersion == 3) { + if (ptx->version == TRUC_VERSION) { + // SingleV3Checks should have checked this already. + if (!Assume(vsize <= V3_MAX_VSIZE)) { + return strprintf("v3 tx %s (wtxid=%s) is too big: %u > %u virtual bytes", + ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString(), vsize, V3_MAX_VSIZE); + } + if (mempool_ancestors.size() + in_package_parents.size() + 1 > V3_ANCESTOR_LIMIT) { return strprintf("tx %s (wtxid=%s) would have too many ancestors", ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString()); @@ -85,23 +91,22 @@ std::optional<std::string> PackageV3Checks(const CTransactionRef& ptx, int64_t v const auto parent_info = [&] { if (mempool_ancestors.size() > 0) { auto& mempool_parent = *mempool_ancestors.begin(); - Assume(mempool_parent->GetCountWithDescendants() == 1); return ParentInfo{mempool_parent->GetTx().GetHash(), mempool_parent->GetTx().GetWitnessHash(), - mempool_parent->GetTx().nVersion, + mempool_parent->GetTx().version, /*has_mempool_descendant=*/mempool_parent->GetCountWithDescendants() > 1}; } else { auto& parent_index = in_package_parents.front(); auto& package_parent = package.at(parent_index); return ParentInfo{package_parent->GetHash(), package_parent->GetWitnessHash(), - package_parent->nVersion, + package_parent->version, /*has_mempool_descendant=*/false}; } }(); // If there is a parent, it must have the right version. - if (parent_info.m_version != 3) { + if (parent_info.m_version != TRUC_VERSION) { return strprintf("v3 tx %s (wtxid=%s) cannot spend from non-v3 tx %s (wtxid=%s)", ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString(), parent_info.m_txid.ToString(), parent_info.m_wtxid.ToString()); @@ -129,10 +134,7 @@ std::optional<std::string> PackageV3Checks(const CTransactionRef& ptx, int64_t v } } - // It shouldn't be possible to have any mempool siblings at this point. SingleV3Checks - // catches mempool siblings and sibling eviction is not extended to packages. Also, if the package consists of connected transactions, - // any tx having a mempool ancestor would mean the package exceeds ancestor limits. - if (!Assume(!parent_info.m_has_mempool_descendant)) { + if (parent_info.m_has_mempool_descendant) { return strprintf("tx %s (wtxid=%s) would exceed descendant count limit", parent_info.m_txid.ToString(), parent_info.m_wtxid.ToString()); } @@ -140,14 +142,14 @@ std::optional<std::string> PackageV3Checks(const CTransactionRef& ptx, int64_t v } else { // Non-v3 transactions cannot have v3 parents. for (auto it : mempool_ancestors) { - if (it->GetTx().nVersion == 3) { + if (it->GetTx().version == TRUC_VERSION) { return strprintf("non-v3 tx %s (wtxid=%s) cannot spend from v3 tx %s (wtxid=%s)", ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString(), it->GetSharedTx()->GetHash().ToString(), it->GetSharedTx()->GetWitnessHash().ToString()); } } for (const auto& index: in_package_parents) { - if (package.at(index)->nVersion == 3) { + if (package.at(index)->version == TRUC_VERSION) { return strprintf("non-v3 tx %s (wtxid=%s) cannot spend from v3 tx %s (wtxid=%s)", ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString(), @@ -166,12 +168,12 @@ std::optional<std::pair<std::string, CTransactionRef>> SingleV3Checks(const CTra { // Check v3 and non-v3 inheritance. for (const auto& entry : mempool_ancestors) { - if (ptx->nVersion != 3 && entry->GetTx().nVersion == 3) { + if (ptx->version != TRUC_VERSION && entry->GetTx().version == TRUC_VERSION) { return std::make_pair(strprintf("non-v3 tx %s (wtxid=%s) cannot spend from v3 tx %s (wtxid=%s)", ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString(), entry->GetSharedTx()->GetHash().ToString(), entry->GetSharedTx()->GetWitnessHash().ToString()), nullptr); - } else if (ptx->nVersion == 3 && entry->GetTx().nVersion != 3) { + } else if (ptx->version == TRUC_VERSION && entry->GetTx().version != TRUC_VERSION) { return std::make_pair(strprintf("v3 tx %s (wtxid=%s) cannot spend from non-v3 tx %s (wtxid=%s)", ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString(), entry->GetSharedTx()->GetHash().ToString(), entry->GetSharedTx()->GetWitnessHash().ToString()), @@ -183,8 +185,14 @@ std::optional<std::pair<std::string, CTransactionRef>> SingleV3Checks(const CTra static_assert(V3_ANCESTOR_LIMIT == 2); static_assert(V3_DESCENDANT_LIMIT == 2); - // The rest of the rules only apply to transactions with nVersion=3. - if (ptx->nVersion != 3) return std::nullopt; + // The rest of the rules only apply to transactions with version=3. + if (ptx->version != TRUC_VERSION) return std::nullopt; + + if (vsize > V3_MAX_VSIZE) { + return std::make_pair(strprintf("v3 tx %s (wtxid=%s) is too big: %u > %u virtual bytes", + ptx->GetHash().ToString(), ptx->GetWitnessHash().ToString(), vsize, V3_MAX_VSIZE), + nullptr); + } // Check that V3_ANCESTOR_LIMIT would not be violated. if (mempool_ancestors.size() + 1 > V3_ANCESTOR_LIMIT) { diff --git a/src/policy/v3_policy.h b/src/policy/v3_policy.h index 2e56f8822b..90eaeda46f 100644 --- a/src/policy/v3_policy.h +++ b/src/policy/v3_policy.h @@ -15,8 +15,9 @@ #include <set> #include <string> -// This module enforces rules for transactions with nVersion=3 ("v3 transactions") which help make +// This module enforces rules for BIP 431 TRUC transactions (with version=3) which help make // RBF abilities more robust. +static constexpr decltype(CTransaction::version) TRUC_VERSION{3}; // v3 only allows 1 parent and 1 child when unconfirmed. /** Maximum number of transactions including an unconfirmed tx and its descendants. */ @@ -24,11 +25,13 @@ static constexpr unsigned int V3_DESCENDANT_LIMIT{2}; /** Maximum number of transactions including a V3 tx and all its mempool ancestors. */ static constexpr unsigned int V3_ANCESTOR_LIMIT{2}; +/** Maximum sigop-adjusted virtual size of all v3 transactions. */ +static constexpr int64_t V3_MAX_VSIZE{10000}; /** Maximum sigop-adjusted virtual size of a tx which spends from an unconfirmed v3 transaction. */ static constexpr int64_t V3_CHILD_MAX_VSIZE{1000}; // These limits are within the default ancestor/descendant limits. -static_assert(V3_CHILD_MAX_VSIZE + MAX_STANDARD_TX_WEIGHT / WITNESS_SCALE_FACTOR <= DEFAULT_ANCESTOR_SIZE_LIMIT_KVB * 1000); -static_assert(V3_CHILD_MAX_VSIZE + MAX_STANDARD_TX_WEIGHT / WITNESS_SCALE_FACTOR <= DEFAULT_DESCENDANT_SIZE_LIMIT_KVB * 1000); +static_assert(V3_MAX_VSIZE + V3_CHILD_MAX_VSIZE <= DEFAULT_ANCESTOR_SIZE_LIMIT_KVB * 1000); +static_assert(V3_MAX_VSIZE + V3_CHILD_MAX_VSIZE <= DEFAULT_DESCENDANT_SIZE_LIMIT_KVB * 1000); /** Must be called for every transaction, even if not v3. Not strictly necessary for transactions * accepted through AcceptMultipleTransactions. @@ -40,6 +43,7 @@ static_assert(V3_CHILD_MAX_VSIZE + MAX_STANDARD_TX_WEIGHT / WITNESS_SCALE_FACTOR * 4. A v3's descendant set, including itself, must be within V3_DESCENDANT_LIMIT. * 5. If a v3 tx has any unconfirmed ancestors, the tx's sigop-adjusted vsize must be within * V3_CHILD_MAX_VSIZE. + * 6. A v3 tx must be within V3_MAX_VSIZE. * * * @param[in] mempool_ancestors The in-mempool ancestors of ptx. diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index b4a860dd9e..fab5c40765 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -6,12 +6,12 @@ #include <primitives/transaction.h> #include <consensus/amount.h> +#include <crypto/hex_base.h> #include <hash.h> #include <script/script.h> #include <serialize.h> #include <tinyformat.h> #include <uint256.h> -#include <util/strencodings.h> #include <util/transaction_identifier.h> #include <algorithm> @@ -63,8 +63,8 @@ std::string CTxOut::ToString() const return strprintf("CTxOut(nValue=%d.%08d, scriptPubKey=%s)", nValue / COIN, nValue % COIN, HexStr(scriptPubKey).substr(0, 30)); } -CMutableTransaction::CMutableTransaction() : nVersion(CTransaction::CURRENT_VERSION), nLockTime(0) {} -CMutableTransaction::CMutableTransaction(const CTransaction& tx) : vin(tx.vin), vout(tx.vout), nVersion(tx.nVersion), nLockTime(tx.nLockTime) {} +CMutableTransaction::CMutableTransaction() : version{CTransaction::CURRENT_VERSION}, nLockTime{0} {} +CMutableTransaction::CMutableTransaction(const CTransaction& tx) : vin(tx.vin), vout(tx.vout), version{tx.version}, nLockTime{tx.nLockTime} {} Txid CMutableTransaction::GetHash() const { @@ -92,8 +92,8 @@ Wtxid CTransaction::ComputeWitnessHash() const return Wtxid::FromUint256((HashWriter{} << TX_WITH_WITNESS(*this)).GetHash()); } -CTransaction::CTransaction(const CMutableTransaction& tx) : vin(tx.vin), vout(tx.vout), nVersion(tx.nVersion), nLockTime(tx.nLockTime), m_has_witness{ComputeHasWitness()}, hash{ComputeHash()}, m_witness_hash{ComputeWitnessHash()} {} -CTransaction::CTransaction(CMutableTransaction&& tx) : vin(std::move(tx.vin)), vout(std::move(tx.vout)), nVersion(tx.nVersion), nLockTime(tx.nLockTime), m_has_witness{ComputeHasWitness()}, hash{ComputeHash()}, m_witness_hash{ComputeWitnessHash()} {} +CTransaction::CTransaction(const CMutableTransaction& tx) : vin(tx.vin), vout(tx.vout), version{tx.version}, nLockTime{tx.nLockTime}, m_has_witness{ComputeHasWitness()}, hash{ComputeHash()}, m_witness_hash{ComputeWitnessHash()} {} +CTransaction::CTransaction(CMutableTransaction&& tx) : vin(std::move(tx.vin)), vout(std::move(tx.vout)), version{tx.version}, nLockTime{tx.nLockTime}, m_has_witness{ComputeHasWitness()}, hash{ComputeHash()}, m_witness_hash{ComputeWitnessHash()} {} CAmount CTransaction::GetValueOut() const { @@ -115,9 +115,9 @@ unsigned int CTransaction::GetTotalSize() const std::string CTransaction::ToString() const { std::string str; - str += strprintf("CTransaction(hash=%s, ver=%d, vin.size=%u, vout.size=%u, nLockTime=%u)\n", + str += strprintf("CTransaction(hash=%s, ver=%u, vin.size=%u, vout.size=%u, nLockTime=%u)\n", GetHash().ToString().substr(0,10), - nVersion, + version, vin.size(), vout.size(), nLockTime); diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 976542cfae..bf86562886 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -197,13 +197,13 @@ static constexpr TransactionSerParams TX_NO_WITNESS{.allow_witness = false}; /** * Basic transaction serialization format: - * - int32_t nVersion + * - uint32_t version * - std::vector<CTxIn> vin * - std::vector<CTxOut> vout * - uint32_t nLockTime * * Extended transaction serialization format: - * - int32_t nVersion + * - uint32_t version * - unsigned char dummy = 0x00 * - unsigned char flags (!= 0) * - std::vector<CTxIn> vin @@ -217,7 +217,7 @@ void UnserializeTransaction(TxType& tx, Stream& s, const TransactionSerParams& p { const bool fAllowWitness = params.allow_witness; - s >> tx.nVersion; + s >> tx.version; unsigned char flags = 0; tx.vin.clear(); tx.vout.clear(); @@ -257,7 +257,7 @@ void SerializeTransaction(const TxType& tx, Stream& s, const TransactionSerParam { const bool fAllowWitness = params.allow_witness; - s << tx.nVersion; + s << tx.version; unsigned char flags = 0; // Consistency check if (fAllowWitness) { @@ -296,7 +296,7 @@ class CTransaction { public: // Default transaction version. - static const int32_t CURRENT_VERSION=2; + static const uint32_t CURRENT_VERSION{2}; // The local variables are made const to prevent unintended modification // without updating the cached hash value. However, CTransaction is not @@ -305,7 +305,7 @@ public: // structure, including the hash. const std::vector<CTxIn> vin; const std::vector<CTxOut> vout; - const int32_t nVersion; + const uint32_t version; const uint32_t nLockTime; private: @@ -378,7 +378,7 @@ struct CMutableTransaction { std::vector<CTxIn> vin; std::vector<CTxOut> vout; - int32_t nVersion; + uint32_t version; uint32_t nLockTime; explicit CMutableTransaction(); diff --git a/src/psbt.cpp b/src/psbt.cpp index b2ee3ce7a5..19d855e4c7 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -4,12 +4,12 @@ #include <psbt.h> +#include <node/types.h> #include <policy/policy.h> #include <script/signingprovider.h> #include <util/check.h> #include <util/strencodings.h> - PartiallySignedTransaction::PartiallySignedTransaction(const CMutableTransaction& tx) : tx(tx) { inputs.resize(tx.vin.size()); @@ -508,17 +508,17 @@ bool FinalizeAndExtractPSBT(PartiallySignedTransaction& psbtx, CMutableTransacti return true; } -TransactionError CombinePSBTs(PartiallySignedTransaction& out, const std::vector<PartiallySignedTransaction>& psbtxs) +bool CombinePSBTs(PartiallySignedTransaction& out, const std::vector<PartiallySignedTransaction>& psbtxs) { out = psbtxs[0]; // Copy the first one // Merge for (auto it = std::next(psbtxs.begin()); it != psbtxs.end(); ++it) { if (!out.Merge(*it)) { - return TransactionError::PSBT_MISMATCH; + return false; } } - return TransactionError::OK; + return true; } std::string PSBTRoleName(PSBTRole role) { diff --git a/src/psbt.h b/src/psbt.h index 3f74083717..4607304046 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -17,6 +17,10 @@ #include <optional> +namespace node { +enum class TransactionError; +} // namespace node + // Magic bytes static constexpr uint8_t PSBT_MAGIC_BYTES[5] = {'p', 's', 'b', 't', 0xff}; @@ -237,7 +241,7 @@ struct PSBTInput if (final_script_sig.empty() && final_script_witness.IsNull()) { // Write any partial signatures - for (auto sig_pair : partial_sigs) { + for (const auto& sig_pair : partial_sigs) { SerializeToVector(s, CompactSizeWriter(PSBT_IN_PARTIAL_SIG), Span{sig_pair.second.first}); s << sig_pair.second.second; } @@ -1263,9 +1267,9 @@ bool FinalizeAndExtractPSBT(PartiallySignedTransaction& psbtx, CMutableTransacti * * @param[out] out the combined PSBT, if successful * @param[in] psbtxs the PSBTs to combine - * @return error (OK if we successfully combined the transactions, other error if they were not compatible) + * @return True if we successfully combined the transactions, false if they were not compatible */ -[[nodiscard]] TransactionError CombinePSBTs(PartiallySignedTransaction& out, const std::vector<PartiallySignedTransaction>& psbtxs); +[[nodiscard]] bool CombinePSBTs(PartiallySignedTransaction& out, const std::vector<PartiallySignedTransaction>& psbtxs); //! Decode a base64ed PSBT into a PartiallySignedTransaction [[nodiscard]] bool DecodeBase64PSBT(PartiallySignedTransaction& decoded_psbt, const std::string& base64_psbt, std::string& error); diff --git a/src/qt/android/.gitignore b/src/qt/android/.gitignore deleted file mode 100644 index c090a2e98e..0000000000 --- a/src/qt/android/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -/.gradle -/build -/gradle -/gradlew* -/libs -/src/org/kde -/src/org/qtproject diff --git a/src/qt/android/AndroidManifest.xml b/src/qt/android/AndroidManifest.xml deleted file mode 100644 index 41615294e0..0000000000 --- a/src/qt/android/AndroidManifest.xml +++ /dev/null @@ -1,39 +0,0 @@ -<?xml version='1.0' encoding='utf-8'?> -<manifest package="org.bitcoincore.qt" xmlns:android="http://schemas.android.com/apk/res/android" android:versionName="1.0" android:versionCode="1" android:installLocation="auto"> - <uses-sdk android:targetSdkVersion="24"/> - - <uses-permission android:name="android.permission.INTERNET" /> - <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" /> - <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" /> - - <uses-feature android:glEsVersion="0x00020000" android:required="true" /> - - <supports-screens android:largeScreens="true" android:normalScreens="true" android:anyDensity="true" android:smallScreens="true"/> - - <application android:hardwareAccelerated="true" android:name="org.qtproject.qt5.android.bindings.QtApplication" android:label="Bitcoin Core"> - <activity android:configChanges="orientation|uiMode|screenLayout|screenSize|smallestScreenSize|layoutDirection|locale|fontScale|keyboard|keyboardHidden|navigation|mcc|mnc|density" - android:name="org.bitcoincore.qt.BitcoinQtActivity" - android:label="Bitcoin Core" - android:icon="@drawable/bitcoin" - android:screenOrientation="unspecified" - android:launchMode="singleTop"> - <intent-filter> - <action android:name="android.intent.action.MAIN"/> - <category android:name="android.intent.category.LAUNCHER"/> - </intent-filter> - - <meta-data android:name="android.app.arguments" android:value="-testnet"/> - <meta-data android:name="android.app.lib_name" android:value="bitcoin-qt"/> - <meta-data android:name="android.app.repository" android:value="default"/> - <meta-data android:name="android.app.bundle_local_qt_libs" android:value="1"/> - <meta-data android:name="android.app.use_local_qt_libs" android:value="1"/> - <meta-data android:name="android.app.libs_prefix" android:value="/data/local/tmp/qt/"/> - <meta-data android:name="android.app.system_libs_prefix" android:value="/system/lib/"/> - <meta-data android:name="android.app.background_running" android:value="true"/> - <meta-data android:name="android.app.auto_screen_scale_factor" android:value="true"/> - <meta-data android:name="android.app.extract_android_style" android:value="default"/> - <meta-data android:name="android.app.load_local_libs_resource_id" android:resource="@array/load_local_libs"/> - </activity> - - </application> -</manifest> diff --git a/src/qt/android/build.gradle b/src/qt/android/build.gradle deleted file mode 100644 index 4c36e79db8..0000000000 --- a/src/qt/android/build.gradle +++ /dev/null @@ -1,52 +0,0 @@ -buildscript { - repositories { - google() - jcenter() - } - - dependencies { - classpath 'com.android.tools.build:gradle:3.1.0' - } -} - -repositories { - google() - jcenter() -} - -apply plugin: 'com.android.application' - -dependencies { - implementation fileTree(dir: 'libs', include: ['*.jar']) -} - -android { - compileSdkVersion androidCompileSdkVersion.toInteger() - - buildToolsVersion androidBuildToolsVersion - - sourceSets { - main { - manifest.srcFile 'AndroidManifest.xml' - java.srcDirs = [qt5AndroidDir + '/src', 'src', 'java'] - aidl.srcDirs = [qt5AndroidDir + '/src', 'src', 'aidl'] - res.srcDirs = [qt5AndroidDir + '/res', 'res'] - resources.srcDirs = ['src'] - renderscript.srcDirs = ['src'] - assets.srcDirs = ['assets'] - jniLibs.srcDirs = ['libs'] - } - } - - lintOptions { - abortOnError false - } - - dexOptions { - javaMaxHeapSize '4g' - } - - defaultConfig { - minSdkVersion 24 - } -} diff --git a/src/qt/android/gradle.properties b/src/qt/android/gradle.properties deleted file mode 100644 index 468f4757c0..0000000000 --- a/src/qt/android/gradle.properties +++ /dev/null @@ -1,4 +0,0 @@ -androidBuildToolsVersion=30.0.3 -androidCompileSdkVersion=30 -qt5AndroidDir=new File(".").absolutePath -org.gradle.jvmargs=-Xmx4608M diff --git a/src/qt/android/res/drawable-hdpi/bitcoin.png b/src/qt/android/res/drawable-hdpi/bitcoin.png Binary files differdeleted file mode 100644 index 31a556a35f..0000000000 --- a/src/qt/android/res/drawable-hdpi/bitcoin.png +++ /dev/null diff --git a/src/qt/android/res/drawable-ldpi/bitcoin.png b/src/qt/android/res/drawable-ldpi/bitcoin.png Binary files differdeleted file mode 100644 index 76d80d4196..0000000000 --- a/src/qt/android/res/drawable-ldpi/bitcoin.png +++ /dev/null diff --git a/src/qt/android/res/drawable-mdpi/bitcoin.png b/src/qt/android/res/drawable-mdpi/bitcoin.png Binary files differdeleted file mode 100644 index c2aeab851a..0000000000 --- a/src/qt/android/res/drawable-mdpi/bitcoin.png +++ /dev/null diff --git a/src/qt/android/res/drawable-xhdpi/bitcoin.png b/src/qt/android/res/drawable-xhdpi/bitcoin.png Binary files differdeleted file mode 100644 index 2bd5e3defc..0000000000 --- a/src/qt/android/res/drawable-xhdpi/bitcoin.png +++ /dev/null diff --git a/src/qt/android/res/drawable-xxhdpi/bitcoin.png b/src/qt/android/res/drawable-xxhdpi/bitcoin.png Binary files differdeleted file mode 100644 index d236cf2132..0000000000 --- a/src/qt/android/res/drawable-xxhdpi/bitcoin.png +++ /dev/null diff --git a/src/qt/android/res/drawable-xxxhdpi/bitcoin.png b/src/qt/android/res/drawable-xxxhdpi/bitcoin.png Binary files differdeleted file mode 100644 index bb1dbc3554..0000000000 --- a/src/qt/android/res/drawable-xxxhdpi/bitcoin.png +++ /dev/null diff --git a/src/qt/android/res/values/libs.xml b/src/qt/android/res/values/libs.xml deleted file mode 100644 index b4b77b1c7b..0000000000 --- a/src/qt/android/res/values/libs.xml +++ /dev/null @@ -1,14 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<resources> - <array name="load_local_libs"> - <item> - arm64-v8a;libbitcoin-qt_arm64-v8a.so - </item> - <item> - armeabi-v7a;libbitcoin-qt_armeabi-v7a.so - </item> - <item> - x86_64;libbitcoin-qt_x86_64.so - </item> - </array> -</resources> diff --git a/src/qt/android/src/org/bitcoincore/qt/BitcoinQtActivity.java b/src/qt/android/src/org/bitcoincore/qt/BitcoinQtActivity.java deleted file mode 100644 index 2cba489242..0000000000 --- a/src/qt/android/src/org/bitcoincore/qt/BitcoinQtActivity.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.bitcoincore.qt; - -import android.os.Bundle; -import android.system.ErrnoException; -import android.system.Os; - -import org.qtproject.qt5.android.bindings.QtActivity; - -import java.io.File; - -public class BitcoinQtActivity extends QtActivity -{ - @Override - public void onCreate(Bundle savedInstanceState) - { - final File bitcoinDir = new File(getFilesDir().getAbsolutePath() + "/.bitcoin"); - if (!bitcoinDir.exists()) { - bitcoinDir.mkdir(); - } - - super.onCreate(savedInstanceState); - } -} diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 44a858c16b..6c5725533b 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -85,6 +85,8 @@ Q_DECLARE_METATYPE(uint256) Q_DECLARE_METATYPE(wallet::AddressPurpose) #endif // ENABLE_WALLET +using util::MakeUnorderedList; + static void RegisterMetaTypes() { // Register meta types used for QMetaObject::invokeMethod and Qt::QueuedConnection diff --git a/src/qt/psbtoperationsdialog.cpp b/src/qt/psbtoperationsdialog.cpp index 353709c7f5..5a4b4442f3 100644 --- a/src/qt/psbtoperationsdialog.cpp +++ b/src/qt/psbtoperationsdialog.cpp @@ -4,10 +4,12 @@ #include <qt/psbtoperationsdialog.h> +#include <common/messages.h> #include <core_io.h> #include <interfaces/node.h> #include <key_io.h> #include <node/psbt.h> +#include <node/types.h> #include <policy/policy.h> #include <qt/bitcoinunits.h> #include <qt/forms/ui_psbtoperationsdialog.h> @@ -20,9 +22,11 @@ #include <iostream> #include <string> +using common::TransactionErrorString; using node::AnalyzePSBT; using node::DEFAULT_MAX_RAW_TX_FEE_RATE; using node::PSBTAnalysis; +using node::TransactionError; PSBTOperationsDialog::PSBTOperationsDialog( QWidget* parent, WalletModel* wallet_model, ClientModel* client_model) : QDialog(parent, GUIUtil::dialog_flags), @@ -55,10 +59,10 @@ void PSBTOperationsDialog::openWithPSBT(PartiallySignedTransaction psbtx) bool complete = FinalizePSBT(psbtx); // Make sure all existing signatures are fully combined before checking for completeness. if (m_wallet_model) { size_t n_could_sign; - TransactionError err = m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, &n_could_sign, m_transaction_data, complete); - if (err != TransactionError::OK) { + const auto err{m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, &n_could_sign, m_transaction_data, complete)}; + if (err) { showStatus(tr("Failed to load transaction: %1") - .arg(QString::fromStdString(TransactionErrorString(err).translated)), + .arg(QString::fromStdString(PSBTErrorString(*err).translated)), StatusLevel::ERR); return; } @@ -79,11 +83,11 @@ void PSBTOperationsDialog::signTransaction() WalletModel::UnlockContext ctx(m_wallet_model->requestUnlock()); - TransactionError err = m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/true, /*bip32derivs=*/true, &n_signed, m_transaction_data, complete); + const auto err{m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/true, /*bip32derivs=*/true, &n_signed, m_transaction_data, complete)}; - if (err != TransactionError::OK) { + if (err) { showStatus(tr("Failed to sign transaction: %1") - .arg(QString::fromStdString(TransactionErrorString(err).translated)), StatusLevel::ERR); + .arg(QString::fromStdString(PSBTErrorString(*err).translated)), StatusLevel::ERR); return; } @@ -247,9 +251,9 @@ size_t PSBTOperationsDialog::couldSignInputs(const PartiallySignedTransaction &p size_t n_signed; bool complete; - TransactionError err = m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/false, &n_signed, m_transaction_data, complete); + const auto err{m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/false, &n_signed, m_transaction_data, complete)}; - if (err != TransactionError::OK) { + if (err) { return 0; } return n_signed; diff --git a/src/qt/recentrequeststablemodel.cpp b/src/qt/recentrequeststablemodel.cpp index 52d4e45d49..a8d54cdc0c 100644 --- a/src/qt/recentrequeststablemodel.cpp +++ b/src/qt/recentrequeststablemodel.cpp @@ -20,6 +20,8 @@ #include <QLatin1Char> #include <QLatin1String> +using util::ToString; + RecentRequestsTableModel::RecentRequestsTableModel(WalletModel *parent) : QAbstractTableModel(parent), walletModel(parent) { diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 702ca44395..edf417a7cb 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -48,6 +48,8 @@ #include <chrono> +using util::Join; + const int CONSOLE_HISTORY = 50; const int INITIAL_TRAFFIC_GRAPH_MINS = 30; const QSize FONT_RANGE(4, 40); diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 0d8c0f7a63..03173ec80e 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -20,6 +20,7 @@ #include <interfaces/node.h> #include <key_io.h> #include <node/interface_ui.h> +#include <node/types.h> #include <policy/fees.h> #include <txmempool.h> #include <validation.h> @@ -37,6 +38,7 @@ #include <QSettings> #include <QTextDocument> +using common::PSBTError; using wallet::CCoinControl; using wallet::DEFAULT_PAY_TX_FEE; @@ -442,26 +444,26 @@ void SendCoinsDialog::presentPSBT(PartiallySignedTransaction& psbtx) } bool SendCoinsDialog::signWithExternalSigner(PartiallySignedTransaction& psbtx, CMutableTransaction& mtx, bool& complete) { - TransactionError err; + std::optional<PSBTError> err; try { err = model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/true, /*bip32derivs=*/true, /*n_signed=*/nullptr, psbtx, complete); } catch (const std::runtime_error& e) { QMessageBox::critical(nullptr, tr("Sign failed"), e.what()); return false; } - if (err == TransactionError::EXTERNAL_SIGNER_NOT_FOUND) { + if (err == PSBTError::EXTERNAL_SIGNER_NOT_FOUND) { //: "External signer" means using devices such as hardware wallets. const QString msg = tr("External signer not found"); QMessageBox::critical(nullptr, msg, msg); return false; } - if (err == TransactionError::EXTERNAL_SIGNER_FAILED) { + if (err == PSBTError::EXTERNAL_SIGNER_FAILED) { //: "External signer" means using devices such as hardware wallets. const QString msg = tr("External signer failure"); QMessageBox::critical(nullptr, msg, msg); return false; } - if (err != TransactionError::OK) { + if (err) { tfm::format(std::cerr, "Failed to sign PSBT"); processSendCoinsReturn(WalletModel::TransactionCreationFailed); return false; @@ -501,9 +503,9 @@ void SendCoinsDialog::sendButtonClicked([[maybe_unused]] bool checked) PartiallySignedTransaction psbtx(mtx); bool complete = false; // Fill without signing - TransactionError err = model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, /*n_signed=*/nullptr, psbtx, complete); + const auto err{model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, /*n_signed=*/nullptr, psbtx, complete)}; assert(!complete); - assert(err == TransactionError::OK); + assert(!err); // Copy PSBT to clipboard and offer to save presentPSBT(psbtx); @@ -517,9 +519,9 @@ void SendCoinsDialog::sendButtonClicked([[maybe_unused]] bool checked) bool complete = false; // Always fill without signing first. This prevents an external signer // from being called prematurely and is not expensive. - TransactionError err = model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, /*n_signed=*/nullptr, psbtx, complete); + const auto err{model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, /*n_signed=*/nullptr, psbtx, complete)}; assert(!complete); - assert(err == TransactionError::OK); + assert(!err); send_failure = !signWithExternalSigner(psbtx, mtx, complete); // Don't broadcast when user rejects it on the device or there's a failure: broadcast = complete && !send_failure; diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index 4392d76328..012186ee4d 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -10,9 +10,9 @@ #include <qt/platformstyle.h> #include <qt/walletmodel.h> +#include <common/signmessage.h> // For MessageSign(), MessageVerify() #include <config/bitcoin-config.h> // IWYU pragma: keep #include <key_io.h> -#include <util/message.h> // For MessageSign(), MessageVerify() #include <wallet/wallet.h> #include <vector> diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index c7fe62f4e9..34b47c90a3 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -30,6 +30,7 @@ #include <QTimer> #include <QWindow> +using util::Join; using wallet::WALLET_FLAG_BLANK_WALLET; using wallet::WALLET_FLAG_DESCRIPTORS; using wallet::WALLET_FLAG_DISABLE_PRIVATE_KEYS; diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 87ad98a4cc..f8ce068e12 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -19,6 +19,7 @@ #include <interfaces/node.h> #include <key_io.h> #include <node/interface_ui.h> +#include <node/types.h> #include <psbt.h> #include <util/translation.h> #include <wallet/coincontrol.h> @@ -534,8 +535,8 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) // "Create Unsigned" clicked PartiallySignedTransaction psbtx(mtx); bool complete = false; - const TransactionError err = wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, nullptr, psbtx, complete); - if (err != TransactionError::OK || complete) { + const auto err{wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, nullptr, psbtx, complete)}; + if (err || complete) { QMessageBox::critical(nullptr, tr("Fee bump error"), tr("Can't draft transaction.")); return false; } diff --git a/src/randomenv.cpp b/src/randomenv.cpp index aeec959c28..49033deef2 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -42,15 +42,15 @@ #if HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS #include <ifaddrs.h> #endif -#if HAVE_SYSCTL +#ifdef HAVE_SYSCTL #include <sys/sysctl.h> -#if HAVE_VM_VM_PARAM_H +#ifdef HAVE_VM_VM_PARAM_H #include <vm/vm_param.h> #endif -#if HAVE_SYS_RESOURCES_H +#ifdef HAVE_SYS_RESOURCES_H #include <sys/resources.h> #endif -#if HAVE_SYS_VMMETER_H +#ifdef HAVE_SYS_VMMETER_H #include <sys/vmmeter.h> #endif #endif @@ -162,7 +162,7 @@ void AddPath(CSHA512& hasher, const char *path) } #endif -#if HAVE_SYSCTL +#ifdef HAVE_SYSCTL template<int... S> void AddSysctl(CSHA512& hasher) { @@ -274,7 +274,7 @@ void RandAddDynamicEnv(CSHA512& hasher) AddFile(hasher, "/proc/self/status"); #endif -#if HAVE_SYSCTL +#ifdef HAVE_SYSCTL # ifdef CTL_KERN # if defined(KERN_PROC) && defined(KERN_PROC_ALL) AddSysctl<CTL_KERN, KERN_PROC, KERN_PROC_ALL>(hasher); @@ -419,7 +419,7 @@ void RandAddStaticEnv(CSHA512& hasher) // For MacOS/BSDs, gather data through sysctl instead of /proc. Not all of these // will exist on every system. -#if HAVE_SYSCTL +#ifdef HAVE_SYSCTL # ifdef CTL_HW # ifdef HW_MACHINE AddSysctl<CTL_HW, HW_MACHINE>(hasher); diff --git a/src/rest.cpp b/src/rest.cpp index 9fc5d4af04..d43018f5af 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -39,6 +39,7 @@ using node::GetTransaction; using node::NodeContext; +using util::SplitString; static const size_t MAX_GETUTXOS_OUTPOINTS = 15; //allow a max of 15 outpoints to be queried at once static constexpr unsigned int MAX_REST_HEADERS_RESULTS = 2000; @@ -934,10 +935,10 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: // include the script in a json output UniValue o(UniValue::VOBJ); ScriptToUniv(coin.out.scriptPubKey, /*out=*/o, /*include_hex=*/true, /*include_address=*/true); - utxo.pushKV("scriptPubKey", o); - utxos.push_back(utxo); + utxo.pushKV("scriptPubKey", std::move(o)); + utxos.push_back(std::move(utxo)); } - objGetUTXOResponse.pushKV("utxos", utxos); + objGetUTXOResponse.pushKV("utxos", std::move(utxos)); // return json string std::string strJSON = objGetUTXOResponse.write() + "\n"; diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 1abaeafb2a..e785678614 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -29,11 +29,13 @@ #include <node/context.h> #include <node/transaction.h> #include <node/utxo_snapshot.h> +#include <node/warnings.h> #include <primitives/transaction.h> #include <rpc/server.h> #include <rpc/server_util.h> #include <rpc/util.h> #include <script/descriptor.h> +#include <serialize.h> #include <streams.h> #include <sync.h> #include <txdb.h> @@ -60,6 +62,9 @@ using kernel::CoinStatsHashType; using node::BlockManager; using node::NodeContext; using node::SnapshotMetadata; +using util::Join; +using util::MakeUnorderedList; +using util::ToString; struct CUpdatedBlock { @@ -1025,9 +1030,9 @@ static RPCHelpMan gettxoutsetinfo() unspendables.pushKV("bip30", ValueFromAmount(stats.total_unspendables_bip30 - prev_stats.total_unspendables_bip30)); unspendables.pushKV("scripts", ValueFromAmount(stats.total_unspendables_scripts - prev_stats.total_unspendables_scripts)); unspendables.pushKV("unclaimed_rewards", ValueFromAmount(stats.total_unspendables_unclaimed_rewards - prev_stats.total_unspendables_unclaimed_rewards)); - block_info.pushKV("unspendables", unspendables); + block_info.pushKV("unspendables", std::move(unspendables)); - ret.pushKV("block_info", block_info); + ret.pushKV("block_info", std::move(block_info)); } } else { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); @@ -1111,7 +1116,7 @@ static RPCHelpMan gettxout() ret.pushKV("value", ValueFromAmount(coin.out.nValue)); UniValue o(UniValue::VOBJ); ScriptToUniv(coin.out.scriptPubKey, /*out=*/o, /*include_hex=*/true, /*include_address=*/true); - ret.pushKV("scriptPubKey", o); + ret.pushKV("scriptPubKey", std::move(o)); ret.pushKV("coinbase", (bool)coin.fCoinBase); return ret; @@ -1161,7 +1166,7 @@ static void SoftForkDescPushBack(const CBlockIndex* blockindex, UniValue& softfo // one below the activation height rv.pushKV("active", DeploymentActiveAfter(blockindex, chainman, dep)); rv.pushKV("height", chainman.GetConsensus().DeploymentHeight(dep)); - softforks.pushKV(DeploymentName(dep), rv); + softforks.pushKV(DeploymentName(dep), std::move(rv)); } static void SoftForkDescPushBack(const CBlockIndex* blockindex, UniValue& softforks, const ChainstateManager& chainman, Consensus::DeploymentPos id) @@ -1214,7 +1219,7 @@ static void SoftForkDescPushBack(const CBlockIndex* blockindex, UniValue& softfo statsUV.pushKV("threshold", statsStruct.threshold); statsUV.pushKV("possible", statsStruct.possible); } - bip9.pushKV("statistics", statsUV); + bip9.pushKV("statistics", std::move(statsUV)); std::string sig; sig.reserve(signals.size()); @@ -1230,9 +1235,9 @@ static void SoftForkDescPushBack(const CBlockIndex* blockindex, UniValue& softfo rv.pushKV("height", chainman.m_versionbitscache.StateSinceHeight(blockindex, chainman.GetConsensus(), id)); } rv.pushKV("active", ThresholdState::ACTIVE == next_state); - rv.pushKV("bip9", bip9); + rv.pushKV("bip9", std::move(bip9)); - softforks.pushKV(DeploymentName(id), rv); + softforks.pushKV(DeploymentName(id), std::move(rv)); } // used by rest.cpp:rest_chaininfo, so cannot be static @@ -1304,7 +1309,8 @@ RPCHelpMan getblockchaininfo() } } - obj.pushKV("warnings", GetNodeWarnings(IsDeprecatedRPCEnabled("warnings"))); + NodeContext& node = EnsureAnyNodeContext(request.context); + obj.pushKV("warnings", node::GetWarningsForRpc(*CHECK_NONFATAL(node.warnings), IsDeprecatedRPCEnabled("warnings"))); return obj; }, }; @@ -1498,7 +1504,7 @@ static RPCHelpMan getchaintips() } obj.pushKV("status", status); - res.push_back(obj); + res.push_back(std::move(obj)); } return res; @@ -1978,7 +1984,7 @@ static RPCHelpMan getblockstats() ret_all.pushKV("avgfeerate", total_weight ? (totalfee * WITNESS_SCALE_FACTOR) / total_weight : 0); // Unit: sat/vbyte ret_all.pushKV("avgtxsize", (block.vtx.size() > 1) ? total_size / (block.vtx.size() - 1) : 0); ret_all.pushKV("blockhash", pindex.GetBlockHash().GetHex()); - ret_all.pushKV("feerate_percentiles", feerates_res); + ret_all.pushKV("feerate_percentiles", std::move(feerates_res)); ret_all.pushKV("height", (int64_t)pindex.nHeight); ret_all.pushKV("ins", inputs); ret_all.pushKV("maxfee", maxfee); @@ -2262,9 +2268,9 @@ static RPCHelpMan scantxoutset() unspent.pushKV("coinbase", coin.IsCoinBase()); unspent.pushKV("height", (int32_t)coin.nHeight); - unspents.push_back(unspent); + unspents.push_back(std::move(unspent)); } - result.pushKV("unspents", unspents); + result.pushKV("unspents", std::move(unspents)); result.pushKV("total_amount", ValueFromAmount(total_in)); } else { throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid action '%s'", action)); @@ -2504,7 +2510,7 @@ static RPCHelpMan scanblocks() ret.pushKV("from_height", start_block_height); ret.pushKV("to_height", start_index->nHeight); // start_index is always the last scanned block here - ret.pushKV("relevant_blocks", blocks); + ret.pushKV("relevant_blocks", std::move(blocks)); ret.pushKV("completed", completed); } else { @@ -2696,29 +2702,60 @@ UniValue CreateUTXOSnapshot( tip->nHeight, tip->GetBlockHash().ToString(), fs::PathToString(path), fs::PathToString(temppath))); - SnapshotMetadata metadata{tip->GetBlockHash(), maybe_stats->coins_count}; + SnapshotMetadata metadata{chainstate.m_chainman.GetParams().MessageStart(), tip->GetBlockHash(), tip->nHeight, maybe_stats->coins_count}; afile << metadata; COutPoint key; + Txid last_hash; Coin coin; unsigned int iter{0}; + size_t written_coins_count{0}; + std::vector<std::pair<uint32_t, Coin>> coins; + + // To reduce space the serialization format of the snapshot avoids + // duplication of tx hashes. The code takes advantage of the guarantee by + // leveldb that keys are lexicographically sorted. + // In the coins vector we collect all coins that belong to a certain tx hash + // (key.hash) and when we have them all (key.hash != last_hash) we write + // them to file using the below lambda function. + // See also https://github.com/bitcoin/bitcoin/issues/25675 + auto write_coins_to_file = [&](AutoFile& afile, const Txid& last_hash, const std::vector<std::pair<uint32_t, Coin>>& coins, size_t& written_coins_count) { + afile << last_hash; + WriteCompactSize(afile, coins.size()); + for (const auto& [n, coin] : coins) { + WriteCompactSize(afile, n); + afile << coin; + ++written_coins_count; + } + }; + pcursor->GetKey(key); + last_hash = key.hash; while (pcursor->Valid()) { if (iter % 5000 == 0) node.rpc_interruption_point(); ++iter; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { - afile << key; - afile << coin; + if (key.hash != last_hash) { + write_coins_to_file(afile, last_hash, coins, written_coins_count); + last_hash = key.hash; + coins.clear(); + } + coins.emplace_back(key.n, coin); } - pcursor->Next(); } + if (!coins.empty()) { + write_coins_to_file(afile, last_hash, coins, written_coins_count); + } + + CHECK_NONFATAL(written_coins_count == maybe_stats->coins_count); + afile.fclose(); UniValue result(UniValue::VOBJ); - result.pushKV("coins_written", maybe_stats->coins_count); + result.pushKV("coins_written", written_coins_count); result.pushKV("base_hash", tip->GetBlockHash().ToString()); result.pushKV("base_height", tip->nHeight); result.pushKV("path", path.utf8string()); @@ -2777,13 +2814,23 @@ static RPCHelpMan loadtxoutset() "Couldn't open file " + path.utf8string() + " for reading."); } - SnapshotMetadata metadata; - afile >> metadata; + SnapshotMetadata metadata{chainman.GetParams().MessageStart()}; + try { + afile >> metadata; + } catch (const std::ios_base::failure& e) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("Unable to parse metadata: %s", e.what())); + } uint256 base_blockhash = metadata.m_base_blockhash; + int base_blockheight = metadata.m_base_blockheight; if (!chainman.GetParams().AssumeutxoForBlockhash(base_blockhash).has_value()) { + auto available_heights = chainman.GetParams().GetAvailableSnapshotHeights(); + std::string heights_formatted = Join(available_heights, ", ", [&](const auto& i) { return ToString(i); }); throw JSONRPCError(RPC_INTERNAL_ERROR, strprintf("Unable to load UTXO snapshot, " - "assumeutxo block hash in snapshot metadata not recognized (%s)", base_blockhash.ToString())); + "assumeutxo block hash in snapshot metadata not recognized (hash: %s, height: %s). The following snapshot heights are available: %s.", + base_blockhash.ToString(), + base_blockheight, + heights_formatted)); } CBlockIndex* snapshot_start_block = WITH_LOCK(::cs_main, return chainman.m_blockman.LookupBlockIndex(base_blockhash)); diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index b8dc148eae..7dfe69a6c5 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -391,7 +391,7 @@ UniValue RPCConvertNamedValues(const std::string &strMethod, const std::vector<s // Use pushKVEnd instead of pushKV to avoid overwriting an explicit // "args" value with an implicit one. Let the RPC server handle the // request as given. - params.pushKVEnd("args", positional_args); + params.pushKVEnd("args", std::move(positional_args)); } return params; diff --git a/src/rpc/external_signer.cpp b/src/rpc/external_signer.cpp index 27c7394909..3ad7a940e0 100644 --- a/src/rpc/external_signer.cpp +++ b/src/rpc/external_signer.cpp @@ -53,13 +53,13 @@ static RPCHelpMan enumeratesigners() UniValue signer_res = UniValue::VOBJ; signer_res.pushKV("fingerprint", signer.m_fingerprint); signer_res.pushKV("name", signer.m_name); - signers_res.push_back(signer_res); + signers_res.push_back(std::move(signer_res)); } } catch (const std::exception& e) { throw JSONRPCError(RPC_MISC_ERROR, e.what()); } UniValue result(UniValue::VOBJ); - result.pushKV("signers", signers_res); + result.pushKV("signers", std::move(signers_res)); return result; } }; diff --git a/src/rpc/fees.cpp b/src/rpc/fees.cpp index b933d8c647..aefe78162b 100644 --- a/src/rpc/fees.cpp +++ b/src/rpc/fees.cpp @@ -3,6 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/messages.h> #include <core_io.h> #include <node/context.h> #include <policy/feerate.h> @@ -14,7 +15,6 @@ #include <rpc/util.h> #include <txmempool.h> #include <univalue.h> -#include <util/fees.h> #include <validationinterface.h> #include <algorithm> @@ -22,6 +22,9 @@ #include <cmath> #include <string> +using common::FeeModeFromString; +using common::FeeModes; +using common::InvalidEstimateModeErrorMessage; using node::NodeContext; static RPCHelpMan estimatesmartfee() @@ -88,7 +91,7 @@ static RPCHelpMan estimatesmartfee() result.pushKV("feerate", ValueFromAmount(feeRate.GetFeePerK())); } else { errors.push_back("Insufficient data or no feerate found"); - result.pushKV("errors", errors); + result.pushKV("errors", std::move(errors)); } result.pushKV("blocks", feeCalc.returnedTarget); return result; @@ -198,18 +201,18 @@ static RPCHelpMan estimaterawfee() horizon_result.pushKV("feerate", ValueFromAmount(feeRate.GetFeePerK())); horizon_result.pushKV("decay", buckets.decay); horizon_result.pushKV("scale", (int)buckets.scale); - horizon_result.pushKV("pass", passbucket); + horizon_result.pushKV("pass", std::move(passbucket)); // buckets.fail.start == -1 indicates that all buckets passed, there is no fail bucket to output - if (buckets.fail.start != -1) horizon_result.pushKV("fail", failbucket); + if (buckets.fail.start != -1) horizon_result.pushKV("fail", std::move(failbucket)); } else { // Output only information that is still meaningful in the event of error horizon_result.pushKV("decay", buckets.decay); horizon_result.pushKV("scale", (int)buckets.scale); - horizon_result.pushKV("fail", failbucket); + horizon_result.pushKV("fail", std::move(failbucket)); errors.push_back("Insufficient data or no feerate found which meets threshold"); - horizon_result.pushKV("errors", errors); + horizon_result.pushKV("errors", std::move(errors)); } - result.pushKV(StringForFeeEstimateHorizon(horizon), horizon_result); + result.pushKV(StringForFeeEstimateHorizon(horizon), std::move(horizon_result)); } return result; }, diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index e599c7dc92..fd11f6cfeb 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -11,6 +11,7 @@ #include <core_io.h> #include <kernel/mempool_entry.h> #include <node/mempool_persist_args.h> +#include <node/types.h> #include <policy/rbf.h> #include <policy/settings.h> #include <primitives/transaction.h> @@ -32,6 +33,8 @@ using node::DEFAULT_MAX_BURN_AMOUNT; using node::DEFAULT_MAX_RAW_TX_FEE_RATE; using node::MempoolPath; using node::NodeContext; +using node::TransactionError; +using util::ToString; static RPCHelpMan sendrawtransaction() { @@ -82,7 +85,7 @@ static RPCHelpMan sendrawtransaction() CTransactionRef tx(MakeTransactionRef(std::move(mtx))); - const CFeeRate max_raw_tx_fee_rate{ParseFeeRate(self.Arg<UniValue>(1))}; + const CFeeRate max_raw_tx_fee_rate{ParseFeeRate(self.Arg<UniValue>("maxfeerate"))}; int64_t virtual_size = GetVirtualTransactionSize(*tx); CAmount max_raw_tx_fee = max_raw_tx_fee_rate.GetFee(virtual_size); @@ -162,7 +165,7 @@ static RPCHelpMan testmempoolaccept() "Array must contain between 1 and " + ToString(MAX_PACKAGE_COUNT) + " transactions."); } - const CFeeRate max_raw_tx_fee_rate{ParseFeeRate(self.Arg<UniValue>(1))}; + const CFeeRate max_raw_tx_fee_rate{ParseFeeRate(self.Arg<UniValue>("maxfeerate"))}; std::vector<CTransactionRef> txns; txns.reserve(raw_transactions.size()); @@ -202,7 +205,7 @@ static RPCHelpMan testmempoolaccept() auto it = package_result.m_tx_results.find(tx->GetWitnessHash()); if (exit_early || it == package_result.m_tx_results.end()) { // Validation unfinished. Just return the txid and wtxid. - rpc_result.push_back(result_inner); + rpc_result.push_back(std::move(result_inner)); continue; } const auto& tx_result = it->second; @@ -229,8 +232,8 @@ static RPCHelpMan testmempoolaccept() for (const auto& wtxid : tx_result.m_wtxids_fee_calculations.value()) { effective_includes_res.push_back(wtxid.ToString()); } - fees.pushKV("effective-includes", effective_includes_res); - result_inner.pushKV("fees", fees); + fees.pushKV("effective-includes", std::move(effective_includes_res)); + result_inner.pushKV("fees", std::move(fees)); } } else { result_inner.pushKV("allowed", false); @@ -241,7 +244,7 @@ static RPCHelpMan testmempoolaccept() result_inner.pushKV("reject-reason", state.GetRejectReason()); } } - rpc_result.push_back(result_inner); + rpc_result.push_back(std::move(result_inner)); } return rpc_result; }, @@ -295,7 +298,7 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool fees.pushKV("modified", ValueFromAmount(e.GetModifiedFee())); fees.pushKV("ancestor", ValueFromAmount(e.GetModFeesWithAncestors())); fees.pushKV("descendant", ValueFromAmount(e.GetModFeesWithDescendants())); - info.pushKV("fees", fees); + info.pushKV("fees", std::move(fees)); const CTransaction& tx = e.GetTx(); std::set<std::string> setDepends; @@ -311,14 +314,14 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool depends.push_back(dep); } - info.pushKV("depends", depends); + info.pushKV("depends", std::move(depends)); UniValue spent(UniValue::VARR); for (const CTxMemPoolEntry& child : e.GetMemPoolChildrenConst()) { spent.push_back(child.GetTx().GetHash().ToString()); } - info.pushKV("spentby", spent); + info.pushKV("spentby", std::move(spent)); // Add opt-in RBF status bool rbfStatus = false; @@ -347,7 +350,7 @@ UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose, bool include_mempoo // Mempool has unique entries so there is no advantage in using // UniValue::pushKV, which checks if the key already exists in O(N). // UniValue::pushKVEnd is used instead which currently is O(1). - o.pushKVEnd(e.GetTx().GetHash().ToString(), info); + o.pushKVEnd(e.GetTx().GetHash().ToString(), std::move(info)); } return o; } else { @@ -364,7 +367,7 @@ UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose, bool include_mempoo return a; } else { UniValue o(UniValue::VOBJ); - o.pushKV("txids", a); + o.pushKV("txids", std::move(a)); o.pushKV("mempool_sequence", mempool_sequence); return o; } @@ -474,7 +477,7 @@ static RPCHelpMan getmempoolancestors() const uint256& _hash = e.GetTx().GetHash(); UniValue info(UniValue::VOBJ); entryToJSON(mempool, info, e); - o.pushKV(_hash.ToString(), info); + o.pushKV(_hash.ToString(), std::move(info)); } return o; } @@ -539,7 +542,7 @@ static RPCHelpMan getmempooldescendants() const uint256& _hash = e.GetTx().GetHash(); UniValue info(UniValue::VOBJ); entryToJSON(mempool, info, e); - o.pushKV(_hash.ToString(), info); + o.pushKV(_hash.ToString(), std::move(info)); } return o; } @@ -653,7 +656,7 @@ static RPCHelpMan gettxspendingprevout() o.pushKV("spendingtxid", spendingTx->GetHash().ToString()); } - result.push_back(o); + result.push_back(std::move(o)); } return result; @@ -873,7 +876,7 @@ static RPCHelpMan submitpackage() } // Fee check needs to be run with chainstate and package context - const CFeeRate max_raw_tx_fee_rate = ParseFeeRate(self.Arg<UniValue>(1)); + const CFeeRate max_raw_tx_fee_rate{ParseFeeRate(self.Arg<UniValue>("maxfeerate"))}; std::optional<CFeeRate> client_maxfeerate{max_raw_tx_fee_rate}; // 0-value is special; it's mapped to no sanity check if (max_raw_tx_fee_rate == CFeeRate(0)) { @@ -992,20 +995,20 @@ static RPCHelpMan submitpackage() for (const auto& wtxid : tx_result.m_wtxids_fee_calculations.value()) { effective_includes_res.push_back(wtxid.ToString()); } - fees.pushKV("effective-includes", effective_includes_res); + fees.pushKV("effective-includes", std::move(effective_includes_res)); } - result_inner.pushKV("fees", fees); + result_inner.pushKV("fees", std::move(fees)); for (const auto& ptx : it->second.m_replaced_transactions) { replaced_txids.insert(ptx->GetHash()); } break; } - tx_result_map.pushKV(tx->GetWitnessHash().GetHex(), result_inner); + tx_result_map.pushKV(tx->GetWitnessHash().GetHex(), std::move(result_inner)); } - rpc_result.pushKV("tx-results", tx_result_map); + rpc_result.pushKV("tx-results", std::move(tx_result_map)); UniValue replaced_list(UniValue::VARR); for (const uint256& hash : replaced_txids) replaced_list.push_back(hash.ToString()); - rpc_result.pushKV("replaced-transactions", replaced_list); + rpc_result.pushKV("replaced-transactions", std::move(replaced_list)); return rpc_result; }, }; diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 2391392bd7..2b93c18965 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -16,10 +16,12 @@ #include <core_io.h> #include <deploymentinfo.h> #include <deploymentstatus.h> +#include <interfaces/mining.h> #include <key_io.h> #include <net.h> #include <node/context.h> #include <node/miner.h> +#include <node/warnings.h> #include <pow.h> #include <rpc/blockchain.h> #include <rpc/mining.h> @@ -44,9 +46,11 @@ using node::BlockAssembler; using node::CBlockTemplate; +using interfaces::Mining; using node::NodeContext; using node::RegenerateCommitments; using node::UpdateTime; +using util::ToString; /** * Return average network hashes per second based on the last 'lookup' blocks, @@ -125,7 +129,7 @@ static RPCHelpMan getnetworkhashps() }; } -static bool GenerateBlock(ChainstateManager& chainman, CBlock& block, uint64_t& max_tries, std::shared_ptr<const CBlock>& block_out, bool process_new_block) +static bool GenerateBlock(ChainstateManager& chainman, Mining& miner, CBlock& block, uint64_t& max_tries, std::shared_ptr<const CBlock>& block_out, bool process_new_block) { block_out.reset(); block.hashMerkleRoot = BlockMerkleRoot(block); @@ -145,23 +149,23 @@ static bool GenerateBlock(ChainstateManager& chainman, CBlock& block, uint64_t& if (!process_new_block) return true; - if (!chainman.ProcessNewBlock(block_out, /*force_processing=*/true, /*min_pow_checked=*/true, nullptr)) { + if (!miner.processNewBlock(block_out, nullptr)) { throw JSONRPCError(RPC_INTERNAL_ERROR, "ProcessNewBlock, block not accepted"); } return true; } -static UniValue generateBlocks(ChainstateManager& chainman, const CTxMemPool& mempool, const CScript& coinbase_script, int nGenerate, uint64_t nMaxTries) +static UniValue generateBlocks(ChainstateManager& chainman, Mining& miner, const CScript& coinbase_script, int nGenerate, uint64_t nMaxTries) { UniValue blockHashes(UniValue::VARR); while (nGenerate > 0 && !chainman.m_interrupt) { - std::unique_ptr<CBlockTemplate> pblocktemplate(BlockAssembler{chainman.ActiveChainstate(), &mempool}.CreateNewBlock(coinbase_script)); + std::unique_ptr<CBlockTemplate> pblocktemplate(miner.createNewBlock(coinbase_script)); if (!pblocktemplate.get()) throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block"); std::shared_ptr<const CBlock> block_out; - if (!GenerateBlock(chainman, pblocktemplate->block, nMaxTries, block_out, /*process_new_block=*/true)) { + if (!GenerateBlock(chainman, miner, pblocktemplate->block, nMaxTries, block_out, /*process_new_block=*/true)) { break; } @@ -237,10 +241,10 @@ static RPCHelpMan generatetodescriptor() } NodeContext& node = EnsureAnyNodeContext(request.context); - const CTxMemPool& mempool = EnsureMemPool(node); + Mining& miner = EnsureMining(node); ChainstateManager& chainman = EnsureChainman(node); - return generateBlocks(chainman, mempool, coinbase_script, num_blocks, max_tries); + return generateBlocks(chainman, miner, coinbase_script, num_blocks, max_tries); }, }; } @@ -283,12 +287,12 @@ static RPCHelpMan generatetoaddress() } NodeContext& node = EnsureAnyNodeContext(request.context); - const CTxMemPool& mempool = EnsureMemPool(node); + Mining& miner = EnsureMining(node); ChainstateManager& chainman = EnsureChainman(node); CScript coinbase_script = GetScriptForDestination(destination); - return generateBlocks(chainman, mempool, coinbase_script, num_blocks, max_tries); + return generateBlocks(chainman, miner, coinbase_script, num_blocks, max_tries); }, }; } @@ -335,6 +339,7 @@ static RPCHelpMan generateblock() } NodeContext& node = EnsureAnyNodeContext(request.context); + Mining& miner = EnsureMining(node); const CTxMemPool& mempool = EnsureMemPool(node); std::vector<CTransactionRef> txs; @@ -368,7 +373,7 @@ static RPCHelpMan generateblock() { LOCK(cs_main); - std::unique_ptr<CBlockTemplate> blocktemplate(BlockAssembler{chainman.ActiveChainstate(), nullptr}.CreateNewBlock(coinbase_script)); + std::unique_ptr<CBlockTemplate> blocktemplate{miner.createNewBlock(coinbase_script, /*use_mempool=*/false)}; if (!blocktemplate) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block"); } @@ -385,15 +390,15 @@ static RPCHelpMan generateblock() LOCK(cs_main); BlockValidationState state; - if (!TestBlockValidity(state, chainman.GetParams(), chainman.ActiveChainstate(), block, chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock), false, false)) { - throw JSONRPCError(RPC_VERIFY_ERROR, strprintf("TestBlockValidity failed: %s", state.ToString())); + if (!miner.testBlockValidity(state, block, /*check_merkle_root=*/false)) { + throw JSONRPCError(RPC_VERIFY_ERROR, strprintf("testBlockValidity failed: %s", state.ToString())); } } std::shared_ptr<const CBlock> block_out; uint64_t max_tries{DEFAULT_MAX_TRIES}; - if (!GenerateBlock(chainman, block, max_tries, block_out, process_new_block) || !block_out) { + if (!GenerateBlock(chainman, miner, block, max_tries, block_out, process_new_block) || !block_out) { throw JSONRPCError(RPC_MISC_ERROR, "Failed to make block."); } @@ -453,7 +458,7 @@ static RPCHelpMan getmininginfo() obj.pushKV("networkhashps", getnetworkhashps().HandleRequest(request)); obj.pushKV("pooledtx", (uint64_t)mempool.size()); obj.pushKV("chain", chainman.GetParams().GetChainTypeString()); - obj.pushKV("warnings", GetNodeWarnings(IsDeprecatedRPCEnabled("warnings"))); + obj.pushKV("warnings", node::GetWarningsForRpc(*CHECK_NONFATAL(node.warnings), IsDeprecatedRPCEnabled("warnings"))); return obj; }, }; @@ -485,7 +490,7 @@ static RPCHelpMan prioritisetransaction() LOCK(cs_main); uint256 hash(ParseHashV(request.params[0], "txid")); - const auto dummy{self.MaybeArg<double>(1)}; + const auto dummy{self.MaybeArg<double>("dummy")}; CAmount nAmount = request.params[2].getInt<int64_t>(); if (dummy && *dummy != 0) { @@ -529,7 +534,7 @@ static RPCHelpMan getprioritisedtransactions() if (delta_info.in_mempool) { result_inner.pushKV("modified_fee", *delta_info.modified_fee); } - rpc_result.pushKV(delta_info.txid.GetHex(), result_inner); + rpc_result.pushKV(delta_info.txid.GetHex(), std::move(result_inner)); } return rpc_result; }, @@ -660,13 +665,15 @@ static RPCHelpMan getblocktemplate() { NodeContext& node = EnsureAnyNodeContext(request.context); ChainstateManager& chainman = EnsureChainman(node); + Mining& miner = EnsureMining(node); LOCK(cs_main); + std::optional<uint256> maybe_tip{miner.getTipHash()}; + CHECK_NONFATAL(maybe_tip); + uint256 tip{maybe_tip.value()}; std::string strMode = "template"; UniValue lpval = NullUniValue; std::set<std::string> setClientRules; - Chainstate& active_chainstate = chainman.ActiveChainstate(); - CChain& active_chain = active_chainstate.m_chain; if (!request.params[0].isNull()) { const UniValue& oparam = request.params[0].get_obj(); @@ -701,12 +708,12 @@ static RPCHelpMan getblocktemplate() return "duplicate-inconclusive"; } - CBlockIndex* const pindexPrev = active_chain.Tip(); - // TestBlockValidity only supports blocks built on the current Tip - if (block.hashPrevBlock != pindexPrev->GetBlockHash()) + // testBlockValidity only supports blocks built on the current Tip + if (block.hashPrevBlock != tip) { return "inconclusive-not-best-prevblk"; + } BlockValidationState state; - TestBlockValidity(state, chainman.GetParams(), active_chainstate, block, pindexPrev, false, true); + miner.testBlockValidity(state, block); return BIP22ValidationResult(state); } @@ -722,19 +729,18 @@ static RPCHelpMan getblocktemplate() if (strMode != "template") throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid mode"); - if (!chainman.GetParams().IsTestChain()) { + if (!miner.isTestChain()) { const CConnman& connman = EnsureConnman(node); if (connman.GetNodeCount(ConnectionDirection::Both) == 0) { throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, PACKAGE_NAME " is not connected!"); } - if (chainman.IsInitialBlockDownload()) { + if (miner.isInitialBlockDownload()) { throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, PACKAGE_NAME " is in initial sync and waiting for blocks..."); } } static unsigned int nTransactionsUpdatedLast; - const CTxMemPool& mempool = EnsureMemPool(node); if (!lpval.isNull()) { @@ -754,7 +760,7 @@ static RPCHelpMan getblocktemplate() else { // NOTE: Spec does not specify behaviour for non-string longpollid, but this makes testing easier - hashWatchedChain = active_chain.Tip()->GetBlockHash(); + hashWatchedChain = tip; nTransactionsUpdatedLastLP = nTransactionsUpdatedLast; } @@ -770,7 +776,7 @@ static RPCHelpMan getblocktemplate() { // Timeout: Check transactions for update // without holding the mempool lock to avoid deadlocks - if (mempool.GetTransactionsUpdated() != nTransactionsUpdatedLastLP) + if (miner.getTransactionsUpdated() != nTransactionsUpdatedLastLP) break; checktxtime += std::chrono::seconds(10); } @@ -778,6 +784,10 @@ static RPCHelpMan getblocktemplate() } ENTER_CRITICAL_SECTION(cs_main); + std::optional<uint256> maybe_tip{miner.getTipHash()}; + CHECK_NONFATAL(maybe_tip); + tip = maybe_tip.value(); + if (!IsRPCRunning()) throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Shutting down"); // TODO: Maybe recheck connections/IBD and (if something wrong) send an expires-immediately template to stop miners? @@ -799,24 +809,25 @@ static RPCHelpMan getblocktemplate() static CBlockIndex* pindexPrev; static int64_t time_start; static std::unique_ptr<CBlockTemplate> pblocktemplate; - if (pindexPrev != active_chain.Tip() || - (mempool.GetTransactionsUpdated() != nTransactionsUpdatedLast && GetTime() - time_start > 5)) + if (!pindexPrev || pindexPrev->GetBlockHash() != tip || + (miner.getTransactionsUpdated() != nTransactionsUpdatedLast && GetTime() - time_start > 5)) { // Clear pindexPrev so future calls make a new block, despite any failures from here on pindexPrev = nullptr; - // Store the pindexBest used before CreateNewBlock, to avoid races - nTransactionsUpdatedLast = mempool.GetTransactionsUpdated(); - CBlockIndex* pindexPrevNew = active_chain.Tip(); + // Store the pindexBest used before createNewBlock, to avoid races + nTransactionsUpdatedLast = miner.getTransactionsUpdated(); + CBlockIndex* pindexPrevNew = chainman.m_blockman.LookupBlockIndex(tip); time_start = GetTime(); // Create new block CScript scriptDummy = CScript() << OP_TRUE; - pblocktemplate = BlockAssembler{active_chainstate, &mempool}.CreateNewBlock(scriptDummy); - if (!pblocktemplate) + pblocktemplate = miner.createNewBlock(scriptDummy); + if (!pblocktemplate) { throw JSONRPCError(RPC_OUT_OF_MEMORY, "Out of memory"); + } - // Need to update only after we know CreateNewBlock succeeded + // Need to update only after we know createNewBlock succeeded pindexPrev = pindexPrevNew; } CHECK_NONFATAL(pindexPrev); @@ -854,7 +865,7 @@ static RPCHelpMan getblocktemplate() if (setTxIndex.count(in.prevout.hash)) deps.push_back(setTxIndex[in.prevout.hash]); } - entry.pushKV("depends", deps); + entry.pushKV("depends", std::move(deps)); int index_in_template = i - 1; entry.pushKV("fee", pblocktemplate->vTxFees[index_in_template]); @@ -866,7 +877,7 @@ static RPCHelpMan getblocktemplate() entry.pushKV("sigops", nTxSigOps); entry.pushKV("weight", GetTransactionWeight(tx)); - transactions.push_back(entry); + transactions.push_back(std::move(entry)); } UniValue aux(UniValue::VOBJ); @@ -879,7 +890,7 @@ static RPCHelpMan getblocktemplate() aMutable.push_back("prevblock"); UniValue result(UniValue::VOBJ); - result.pushKV("capabilities", aCaps); + result.pushKV("capabilities", std::move(aCaps)); UniValue aRules(UniValue::VARR); aRules.push_back("csv"); @@ -931,18 +942,18 @@ static RPCHelpMan getblocktemplate() } } result.pushKV("version", pblock->nVersion); - result.pushKV("rules", aRules); - result.pushKV("vbavailable", vbavailable); + result.pushKV("rules", std::move(aRules)); + result.pushKV("vbavailable", std::move(vbavailable)); result.pushKV("vbrequired", int(0)); result.pushKV("previousblockhash", pblock->hashPrevBlock.GetHex()); - result.pushKV("transactions", transactions); - result.pushKV("coinbaseaux", aux); + result.pushKV("transactions", std::move(transactions)); + result.pushKV("coinbaseaux", std::move(aux)); result.pushKV("coinbasevalue", (int64_t)pblock->vtx[0]->vout[0].nValue); - result.pushKV("longpollid", active_chain.Tip()->GetBlockHash().GetHex() + ToString(nTransactionsUpdatedLast)); + result.pushKV("longpollid", tip.GetHex() + ToString(nTransactionsUpdatedLast)); result.pushKV("target", hashTarget.GetHex()); result.pushKV("mintime", (int64_t)pindexPrev->GetMedianTimePast()+1); - result.pushKV("mutable", aMutable); + result.pushKV("mutable", std::move(aMutable)); result.pushKV("noncerange", "00000000ffffffff"); int64_t nSigOpLimit = MAX_BLOCK_SIGOPS_COST; int64_t nSizeLimit = MAX_BLOCK_SERIALIZED_SIZE; @@ -1045,10 +1056,13 @@ static RPCHelpMan submitblock() } } + NodeContext& node = EnsureAnyNodeContext(request.context); + Mining& miner = EnsureMining(node); + bool new_block; auto sc = std::make_shared<submitblock_StateCatcher>(block.GetHash()); CHECK_NONFATAL(chainman.m_options.signals)->RegisterSharedValidationInterface(sc); - bool accepted = chainman.ProcessNewBlock(blockptr, /*force_processing=*/true, /*min_pow_checked=*/true, /*new_block=*/&new_block); + bool accepted = miner.processNewBlock(blockptr, /*new_block=*/&new_block); CHECK_NONFATAL(chainman.m_options.signals)->UnregisterSharedValidationInterface(sc); if (!new_block && accepted) { return "duplicate"; diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 59397aa84d..1119a3e668 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -16,6 +16,7 @@ #include <netbase.h> #include <node/context.h> #include <node/protocol_version.h> +#include <node/warnings.h> #include <policy/settings.h> #include <protocol.h> #include <rpc/blockchain.h> @@ -35,6 +36,8 @@ #include <univalue.h> using node::NodeContext; +using util::Join; +using util::TrimString; const std::vector<std::string> CONNECTION_TYPE_DOC{ "outbound-full-relay (default automatic connections)", @@ -263,7 +266,7 @@ static RPCHelpMan getpeerinfo() for (const int height : statestats.vHeightInFlight) { heights.push_back(height); } - obj.pushKV("inflight", heights); + obj.pushKV("inflight", std::move(heights)); obj.pushKV("addr_relay_enabled", statestats.m_addr_relay_enabled); obj.pushKV("addr_processed", statestats.m_addr_processed); obj.pushKV("addr_rate_limited", statestats.m_addr_rate_limited); @@ -271,7 +274,7 @@ static RPCHelpMan getpeerinfo() for (const auto& permission : NetPermissions::ToStrings(stats.m_permission_flags)) { permissions.push_back(permission); } - obj.pushKV("permissions", permissions); + obj.pushKV("permissions", std::move(permissions)); obj.pushKV("minfeefilter", ValueFromAmount(statestats.m_fee_filter_received)); UniValue sendPerMsgType(UniValue::VOBJ); @@ -279,19 +282,19 @@ static RPCHelpMan getpeerinfo() if (i.second > 0) sendPerMsgType.pushKV(i.first, i.second); } - obj.pushKV("bytessent_per_msg", sendPerMsgType); + obj.pushKV("bytessent_per_msg", std::move(sendPerMsgType)); UniValue recvPerMsgType(UniValue::VOBJ); for (const auto& i : stats.mapRecvBytesPerMsgType) { if (i.second > 0) recvPerMsgType.pushKV(i.first, i.second); } - obj.pushKV("bytesrecv_per_msg", recvPerMsgType); + obj.pushKV("bytesrecv_per_msg", std::move(recvPerMsgType)); obj.pushKV("connection_type", ConnectionTypeAsString(stats.m_conn_type)); obj.pushKV("transport_protocol_type", TransportTypeAsString(stats.m_transport_type)); obj.pushKV("session_id", stats.m_session_id); - ret.push_back(obj); + ret.push_back(std::move(obj)); } return ret; @@ -401,7 +404,7 @@ static RPCHelpMan addconnection() } else { throw JSONRPCError(RPC_INVALID_PARAMETER, self.ToString()); } - bool use_v2transport = self.Arg<bool>(2); + bool use_v2transport{self.Arg<bool>("v2transport")}; NodeContext& node = EnsureAnyNodeContext(request.context); CConnman& connman = EnsureConnman(node); @@ -532,10 +535,10 @@ static RPCHelpMan getaddednodeinfo() UniValue address(UniValue::VOBJ); address.pushKV("address", info.resolvedAddress.ToStringAddrPort()); address.pushKV("connected", info.fInbound ? "inbound" : "outbound"); - addresses.push_back(address); + addresses.push_back(std::move(address)); } - obj.pushKV("addresses", addresses); - ret.push_back(obj); + obj.pushKV("addresses", std::move(addresses)); + ret.push_back(std::move(obj)); } return ret; @@ -587,7 +590,7 @@ static RPCHelpMan getnettotals() outboundLimit.pushKV("serve_historical_blocks", !connman.OutboundTargetReached(true)); outboundLimit.pushKV("bytes_left_in_cycle", connman.GetOutboundTargetBytesLeft()); outboundLimit.pushKV("time_left_in_cycle", count_seconds(connman.GetMaxOutboundTimeLeftInCycle())); - obj.pushKV("uploadtarget", outboundLimit); + obj.pushKV("uploadtarget", std::move(outboundLimit)); return obj; }, }; @@ -607,7 +610,7 @@ static UniValue GetNetworksInfo() obj.pushKV("reachable", g_reachable_nets.Contains(network)); obj.pushKV("proxy", proxy.IsValid() ? proxy.ToString() : std::string()); obj.pushKV("proxy_randomize_credentials", proxy.m_randomize_credentials); - networks.push_back(obj); + networks.push_back(std::move(obj)); } return networks; } @@ -709,11 +712,11 @@ static RPCHelpMan getnetworkinfo() rec.pushKV("address", item.first.ToStringAddr()); rec.pushKV("port", item.second.nPort); rec.pushKV("score", item.second.nScore); - localAddresses.push_back(rec); + localAddresses.push_back(std::move(rec)); } } - obj.pushKV("localaddresses", localAddresses); - obj.pushKV("warnings", GetNodeWarnings(IsDeprecatedRPCEnabled("warnings"))); + obj.pushKV("localaddresses", std::move(localAddresses)); + obj.pushKV("warnings", node::GetWarningsForRpc(*CHECK_NONFATAL(node.warnings), IsDeprecatedRPCEnabled("warnings"))); return obj; }, }; @@ -843,7 +846,7 @@ static RPCHelpMan listbanned() rec.pushKV("ban_duration", (banEntry.nBanUntil - banEntry.nCreateTime)); rec.pushKV("time_remaining", (banEntry.nBanUntil - current_time)); - bannedAddresses.push_back(rec); + bannedAddresses.push_back(std::move(rec)); } return bannedAddresses; @@ -947,7 +950,7 @@ static RPCHelpMan getnodeaddresses() obj.pushKV("address", addr.ToStringAddr()); obj.pushKV("port", addr.GetPort()); obj.pushKV("network", GetNetworkName(addr.GetNetClass())); - ret.push_back(obj); + ret.push_back(std::move(obj)); } return ret; }, @@ -1087,32 +1090,40 @@ static RPCHelpMan getaddrmaninfo() obj.pushKV("new", addrman.Size(network, true)); obj.pushKV("tried", addrman.Size(network, false)); obj.pushKV("total", addrman.Size(network)); - ret.pushKV(GetNetworkName(network), obj); + ret.pushKV(GetNetworkName(network), std::move(obj)); } UniValue obj(UniValue::VOBJ); obj.pushKV("new", addrman.Size(std::nullopt, true)); obj.pushKV("tried", addrman.Size(std::nullopt, false)); obj.pushKV("total", addrman.Size()); - ret.pushKV("all_networks", obj); + ret.pushKV("all_networks", std::move(obj)); return ret; }, }; } -UniValue AddrmanEntryToJSON(const AddrInfo& info) +UniValue AddrmanEntryToJSON(const AddrInfo& info, CConnman& connman) { UniValue ret(UniValue::VOBJ); ret.pushKV("address", info.ToStringAddr()); + const auto mapped_as{connman.GetMappedAS(info)}; + if (mapped_as != 0) { + ret.pushKV("mapped_as", mapped_as); + } ret.pushKV("port", info.GetPort()); ret.pushKV("services", (uint64_t)info.nServices); ret.pushKV("time", int64_t{TicksSinceEpoch<std::chrono::seconds>(info.nTime)}); ret.pushKV("network", GetNetworkName(info.GetNetClass())); ret.pushKV("source", info.source.ToStringAddr()); ret.pushKV("source_network", GetNetworkName(info.source.GetNetClass())); + const auto source_mapped_as{connman.GetMappedAS(info.source)}; + if (source_mapped_as != 0) { + ret.pushKV("source_mapped_as", source_mapped_as); + } return ret; } -UniValue AddrmanTableToJSON(const std::vector<std::pair<AddrInfo, AddressPosition>>& tableInfos) +UniValue AddrmanTableToJSON(const std::vector<std::pair<AddrInfo, AddressPosition>>& tableInfos, CConnman& connman) { UniValue table(UniValue::VOBJ); for (const auto& e : tableInfos) { @@ -1123,7 +1134,7 @@ UniValue AddrmanTableToJSON(const std::vector<std::pair<AddrInfo, AddressPositio // Address manager tables have unique entries so there is no advantage // in using UniValue::pushKV, which checks if the key already exists // in O(N). UniValue::pushKVEnd is used instead which currently is O(1). - table.pushKVEnd(key.str(), AddrmanEntryToJSON(info)); + table.pushKVEnd(key.str(), AddrmanEntryToJSON(info, connman)); } return table; } @@ -1139,12 +1150,14 @@ static RPCHelpMan getrawaddrman() {RPCResult::Type::OBJ_DYN, "table", "buckets with addresses in the address manager table ( new, tried )", { {RPCResult::Type::OBJ, "bucket/position", "the location in the address manager table (<bucket>/<position>)", { {RPCResult::Type::STR, "address", "The address of the node"}, + {RPCResult::Type::NUM, "mapped_as", /*optional=*/true, "The ASN mapped to the IP of this peer per our current ASMap"}, {RPCResult::Type::NUM, "port", "The port number of the node"}, {RPCResult::Type::STR, "network", "The network (" + Join(GetNetworkNames(), ", ") + ") of the address"}, {RPCResult::Type::NUM, "services", "The services offered by the node"}, {RPCResult::Type::NUM_TIME, "time", "The " + UNIX_EPOCH_TIME + " when the node was last seen"}, {RPCResult::Type::STR, "source", "The address that relayed the address to us"}, {RPCResult::Type::STR, "source_network", "The network (" + Join(GetNetworkNames(), ", ") + ") of the source address"}, + {RPCResult::Type::NUM, "source_mapped_as", /*optional=*/true, "The ASN mapped to the IP of this peer's source per our current ASMap"} }} }} } @@ -1155,10 +1168,12 @@ static RPCHelpMan getrawaddrman() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { AddrMan& addrman = EnsureAnyAddrman(request.context); + NodeContext& node_context = EnsureAnyNodeContext(request.context); + CConnman& connman = EnsureConnman(node_context); UniValue ret(UniValue::VOBJ); - ret.pushKV("new", AddrmanTableToJSON(addrman.GetEntries(false))); - ret.pushKV("tried", AddrmanTableToJSON(addrman.GetEntries(true))); + ret.pushKV("new", AddrmanTableToJSON(addrman.GetEntries(false), connman)); + ret.pushKV("tried", AddrmanTableToJSON(addrman.GetEntries(true), connman)); return ret; }, }; diff --git a/src/rpc/node.cpp b/src/rpc/node.cpp index 447be2cf64..65b0a93cdd 100644 --- a/src/rpc/node.cpp +++ b/src/rpc/node.cpp @@ -351,7 +351,7 @@ static UniValue SummaryToJSON(const IndexSummary&& summary, std::string index_na UniValue entry(UniValue::VOBJ); entry.pushKV("synced", summary.synced); entry.pushKV("best_block_height", summary.best_block_height); - ret_summary.pushKV(summary.name, entry); + ret_summary.pushKV(summary.name, std::move(entry)); return ret_summary; } diff --git a/src/rpc/output_script.cpp b/src/rpc/output_script.cpp index 474d9076be..65a9be2762 100644 --- a/src/rpc/output_script.cpp +++ b/src/rpc/output_script.cpp @@ -72,11 +72,11 @@ static RPCHelpMan validateaddress() ret.pushKV("scriptPubKey", HexStr(scriptPubKey)); UniValue detail = DescribeAddress(dest); - ret.pushKVs(detail); + ret.pushKVs(std::move(detail)); } else { UniValue error_indices(UniValue::VARR); for (int i : error_locations) error_indices.push_back(i); - ret.pushKV("error_locations", error_indices); + ret.pushKV("error_locations", std::move(error_indices)); ret.pushKV("error", error_msg); } @@ -139,8 +139,7 @@ static RPCHelpMan createmultisig() output_type = parsed.value(); } - // Construct using pay-to-script-hash: - FillableSigningProvider keystore; + FlatSigningProvider keystore; CScript inner; const CTxDestination dest = AddAndGetMultisigDestination(required, pubkeys, output_type, keystore, inner); diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 634be2f7fb..75b538061d 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -16,6 +16,7 @@ #include <node/context.h> #include <node/psbt.h> #include <node/transaction.h> +#include <node/types.h> #include <policy/packages.h> #include <policy/policy.h> #include <policy/rbf.h> @@ -618,7 +619,7 @@ static RPCHelpMan decodescript() } ScriptToUniv(segwitScr, /*out=*/sr, /*include_hex=*/true, /*include_address=*/true, /*provider=*/&provider); sr.pushKV("p2sh-segwit", EncodeDestination(ScriptHash(segwitScr))); - r.pushKV("segwit", sr); + r.pushKV("segwit", std::move(sr)); } } @@ -785,7 +786,7 @@ static RPCHelpMan signrawtransactionwithkey() throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed. Make sure the tx has at least one input."); } - FillableSigningProvider keystore; + FlatSigningProvider keystore; const UniValue& keys = request.params[1].get_array(); for (unsigned int idx = 0; idx < keys.size(); ++idx) { UniValue k = keys[idx]; @@ -793,7 +794,11 @@ static RPCHelpMan signrawtransactionwithkey() if (!key.IsValid()) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key"); } - keystore.AddKey(key); + + CPubKey pubkey = key.GetPubKey(); + CKeyID key_id = pubkey.GetID(); + keystore.pubkeys.emplace(key_id, pubkey); + keystore.keys.emplace(key_id, key); } // Fetch previous transactions (inputs): @@ -1069,7 +1074,7 @@ static RPCHelpMan decodepsbt() // Add the decoded tx UniValue tx_univ(UniValue::VOBJ); TxToUniv(CTransaction(*psbtx.tx), /*block_hash=*/uint256(), /*entry=*/tx_univ, /*include_hex=*/false); - result.pushKV("tx", tx_univ); + result.pushKV("tx", std::move(tx_univ)); // Add the global xpubs UniValue global_xpubs(UniValue::VARR); @@ -1083,10 +1088,10 @@ static RPCHelpMan decodepsbt() keypath.pushKV("xpub", EncodeBase58Check(ser_xpub)); keypath.pushKV("master_fingerprint", HexStr(Span<unsigned char>(xpub_pair.first.fingerprint, xpub_pair.first.fingerprint + 4))); keypath.pushKV("path", WriteHDKeypath(xpub_pair.first.path)); - global_xpubs.push_back(keypath); + global_xpubs.push_back(std::move(keypath)); } } - result.pushKV("global_xpubs", global_xpubs); + result.pushKV("global_xpubs", std::move(global_xpubs)); // PSBT version result.pushKV("psbt_version", static_cast<uint64_t>(psbtx.GetVersion())); @@ -1099,16 +1104,16 @@ static RPCHelpMan decodepsbt() this_prop.pushKV("subtype", entry.subtype); this_prop.pushKV("key", HexStr(entry.key)); this_prop.pushKV("value", HexStr(entry.value)); - proprietary.push_back(this_prop); + proprietary.push_back(std::move(this_prop)); } - result.pushKV("proprietary", proprietary); + result.pushKV("proprietary", std::move(proprietary)); // Unknown data UniValue unknowns(UniValue::VOBJ); for (auto entry : psbtx.unknown) { unknowns.pushKV(HexStr(entry.first), HexStr(entry.second)); } - result.pushKV("unknown", unknowns); + result.pushKV("unknown", std::move(unknowns)); // inputs CAmount total_in = 0; @@ -1128,9 +1133,9 @@ static RPCHelpMan decodepsbt() UniValue out(UniValue::VOBJ); out.pushKV("amount", ValueFromAmount(txout.nValue)); - out.pushKV("scriptPubKey", o); + out.pushKV("scriptPubKey", std::move(o)); - in.pushKV("witness_utxo", out); + in.pushKV("witness_utxo", std::move(out)); have_a_utxo = true; } @@ -1139,7 +1144,7 @@ static RPCHelpMan decodepsbt() UniValue non_wit(UniValue::VOBJ); TxToUniv(*input.non_witness_utxo, /*block_hash=*/uint256(), /*entry=*/non_wit, /*include_hex=*/false); - in.pushKV("non_witness_utxo", non_wit); + in.pushKV("non_witness_utxo", std::move(non_wit)); have_a_utxo = true; } @@ -1160,7 +1165,7 @@ static RPCHelpMan decodepsbt() for (const auto& sig : input.partial_sigs) { partial_sigs.pushKV(HexStr(sig.second.first), HexStr(sig.second.second)); } - in.pushKV("partial_signatures", partial_sigs); + in.pushKV("partial_signatures", std::move(partial_sigs)); } // Sighash @@ -1172,12 +1177,12 @@ static RPCHelpMan decodepsbt() if (!input.redeem_script.empty()) { UniValue r(UniValue::VOBJ); ScriptToUniv(input.redeem_script, /*out=*/r); - in.pushKV("redeem_script", r); + in.pushKV("redeem_script", std::move(r)); } if (!input.witness_script.empty()) { UniValue r(UniValue::VOBJ); ScriptToUniv(input.witness_script, /*out=*/r); - in.pushKV("witness_script", r); + in.pushKV("witness_script", std::move(r)); } // keypaths @@ -1189,9 +1194,9 @@ static RPCHelpMan decodepsbt() keypath.pushKV("master_fingerprint", strprintf("%08x", ReadBE32(entry.second.fingerprint))); keypath.pushKV("path", WriteHDKeypath(entry.second.path)); - keypaths.push_back(keypath); + keypaths.push_back(std::move(keypath)); } - in.pushKV("bip32_derivs", keypaths); + in.pushKV("bip32_derivs", std::move(keypaths)); } // Final scriptSig and scriptwitness @@ -1199,14 +1204,14 @@ static RPCHelpMan decodepsbt() UniValue scriptsig(UniValue::VOBJ); scriptsig.pushKV("asm", ScriptToAsmStr(input.final_script_sig, true)); scriptsig.pushKV("hex", HexStr(input.final_script_sig)); - in.pushKV("final_scriptSig", scriptsig); + in.pushKV("final_scriptSig", std::move(scriptsig)); } if (!input.final_script_witness.IsNull()) { UniValue txinwitness(UniValue::VARR); for (const auto& item : input.final_script_witness.stack) { txinwitness.push_back(HexStr(item)); } - in.pushKV("final_scriptwitness", txinwitness); + in.pushKV("final_scriptwitness", std::move(txinwitness)); } // Ripemd160 hash preimages @@ -1215,7 +1220,7 @@ static RPCHelpMan decodepsbt() for (const auto& [hash, preimage] : input.ripemd160_preimages) { ripemd160_preimages.pushKV(HexStr(hash), HexStr(preimage)); } - in.pushKV("ripemd160_preimages", ripemd160_preimages); + in.pushKV("ripemd160_preimages", std::move(ripemd160_preimages)); } // Sha256 hash preimages @@ -1224,7 +1229,7 @@ static RPCHelpMan decodepsbt() for (const auto& [hash, preimage] : input.sha256_preimages) { sha256_preimages.pushKV(HexStr(hash), HexStr(preimage)); } - in.pushKV("sha256_preimages", sha256_preimages); + in.pushKV("sha256_preimages", std::move(sha256_preimages)); } // Hash160 hash preimages @@ -1233,7 +1238,7 @@ static RPCHelpMan decodepsbt() for (const auto& [hash, preimage] : input.hash160_preimages) { hash160_preimages.pushKV(HexStr(hash), HexStr(preimage)); } - in.pushKV("hash160_preimages", hash160_preimages); + in.pushKV("hash160_preimages", std::move(hash160_preimages)); } // Hash256 hash preimages @@ -1242,7 +1247,7 @@ static RPCHelpMan decodepsbt() for (const auto& [hash, preimage] : input.hash256_preimages) { hash256_preimages.pushKV(HexStr(hash), HexStr(preimage)); } - in.pushKV("hash256_preimages", hash256_preimages); + in.pushKV("hash256_preimages", std::move(hash256_preimages)); } // Taproot key path signature @@ -1259,9 +1264,9 @@ static RPCHelpMan decodepsbt() sigobj.pushKV("pubkey", HexStr(xonly)); sigobj.pushKV("leaf_hash", HexStr(leaf_hash)); sigobj.pushKV("sig", HexStr(sig)); - script_sigs.push_back(sigobj); + script_sigs.push_back(std::move(sigobj)); } - in.pushKV("taproot_script_path_sigs", script_sigs); + in.pushKV("taproot_script_path_sigs", std::move(script_sigs)); } // Taproot leaf scripts @@ -1276,10 +1281,10 @@ static RPCHelpMan decodepsbt() for (const auto& control_block : control_blocks) { control_blocks_univ.push_back(HexStr(control_block)); } - script_info.pushKV("control_blocks", control_blocks_univ); - tap_scripts.push_back(script_info); + script_info.pushKV("control_blocks", std::move(control_blocks_univ)); + tap_scripts.push_back(std::move(script_info)); } - in.pushKV("taproot_scripts", tap_scripts); + in.pushKV("taproot_scripts", std::move(tap_scripts)); } // Taproot bip32 keypaths @@ -1295,10 +1300,10 @@ static RPCHelpMan decodepsbt() for (const auto& leaf_hash : leaf_hashes) { leaf_hashes_arr.push_back(HexStr(leaf_hash)); } - path_obj.pushKV("leaf_hashes", leaf_hashes_arr); - keypaths.push_back(path_obj); + path_obj.pushKV("leaf_hashes", std::move(leaf_hashes_arr)); + keypaths.push_back(std::move(path_obj)); } - in.pushKV("taproot_bip32_derivs", keypaths); + in.pushKV("taproot_bip32_derivs", std::move(keypaths)); } // Taproot internal key @@ -1320,9 +1325,9 @@ static RPCHelpMan decodepsbt() this_prop.pushKV("subtype", entry.subtype); this_prop.pushKV("key", HexStr(entry.key)); this_prop.pushKV("value", HexStr(entry.value)); - proprietary.push_back(this_prop); + proprietary.push_back(std::move(this_prop)); } - in.pushKV("proprietary", proprietary); + in.pushKV("proprietary", std::move(proprietary)); } // Unknown data @@ -1331,12 +1336,12 @@ static RPCHelpMan decodepsbt() for (auto entry : input.unknown) { unknowns.pushKV(HexStr(entry.first), HexStr(entry.second)); } - in.pushKV("unknown", unknowns); + in.pushKV("unknown", std::move(unknowns)); } - inputs.push_back(in); + inputs.push_back(std::move(in)); } - result.pushKV("inputs", inputs); + result.pushKV("inputs", std::move(inputs)); // outputs CAmount output_value = 0; @@ -1348,12 +1353,12 @@ static RPCHelpMan decodepsbt() if (!output.redeem_script.empty()) { UniValue r(UniValue::VOBJ); ScriptToUniv(output.redeem_script, /*out=*/r); - out.pushKV("redeem_script", r); + out.pushKV("redeem_script", std::move(r)); } if (!output.witness_script.empty()) { UniValue r(UniValue::VOBJ); ScriptToUniv(output.witness_script, /*out=*/r); - out.pushKV("witness_script", r); + out.pushKV("witness_script", std::move(r)); } // keypaths @@ -1364,9 +1369,9 @@ static RPCHelpMan decodepsbt() keypath.pushKV("pubkey", HexStr(entry.first)); keypath.pushKV("master_fingerprint", strprintf("%08x", ReadBE32(entry.second.fingerprint))); keypath.pushKV("path", WriteHDKeypath(entry.second.path)); - keypaths.push_back(keypath); + keypaths.push_back(std::move(keypath)); } - out.pushKV("bip32_derivs", keypaths); + out.pushKV("bip32_derivs", std::move(keypaths)); } // Taproot internal key @@ -1382,9 +1387,9 @@ static RPCHelpMan decodepsbt() elem.pushKV("depth", (int)depth); elem.pushKV("leaf_ver", (int)leaf_ver); elem.pushKV("script", HexStr(script)); - tree.push_back(elem); + tree.push_back(std::move(elem)); } - out.pushKV("taproot_tree", tree); + out.pushKV("taproot_tree", std::move(tree)); } // Taproot bip32 keypaths @@ -1400,10 +1405,10 @@ static RPCHelpMan decodepsbt() for (const auto& leaf_hash : leaf_hashes) { leaf_hashes_arr.push_back(HexStr(leaf_hash)); } - path_obj.pushKV("leaf_hashes", leaf_hashes_arr); - keypaths.push_back(path_obj); + path_obj.pushKV("leaf_hashes", std::move(leaf_hashes_arr)); + keypaths.push_back(std::move(path_obj)); } - out.pushKV("taproot_bip32_derivs", keypaths); + out.pushKV("taproot_bip32_derivs", std::move(keypaths)); } // Proprietary @@ -1415,9 +1420,9 @@ static RPCHelpMan decodepsbt() this_prop.pushKV("subtype", entry.subtype); this_prop.pushKV("key", HexStr(entry.key)); this_prop.pushKV("value", HexStr(entry.value)); - proprietary.push_back(this_prop); + proprietary.push_back(std::move(this_prop)); } - out.pushKV("proprietary", proprietary); + out.pushKV("proprietary", std::move(proprietary)); } // Unknown data @@ -1426,10 +1431,10 @@ static RPCHelpMan decodepsbt() for (auto entry : output.unknown) { unknowns.pushKV(HexStr(entry.first), HexStr(entry.second)); } - out.pushKV("unknown", unknowns); + out.pushKV("unknown", std::move(unknowns)); } - outputs.push_back(out); + outputs.push_back(std::move(out)); // Fee calculation if (MoneyRange(psbtx.tx->vout[i].nValue) && MoneyRange(output_value + psbtx.tx->vout[i].nValue)) { @@ -1439,7 +1444,7 @@ static RPCHelpMan decodepsbt() have_all_utxos = false; } } - result.pushKV("outputs", outputs); + result.pushKV("outputs", std::move(outputs)); if (have_all_utxos) { result.pushKV("fee", ValueFromAmount(total_in - output_value)); } @@ -1485,9 +1490,8 @@ static RPCHelpMan combinepsbt() } PartiallySignedTransaction merged_psbt; - const TransactionError error = CombinePSBTs(merged_psbt, psbtxs); - if (error != TransactionError::OK) { - throw JSONRPCTransactionError(error); + if (!CombinePSBTs(merged_psbt, psbtxs)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "PSBTs not compatible (different transactions)"); } DataStream ssTx{}; @@ -1744,8 +1748,8 @@ static RPCHelpMan joinpsbts() } psbtxs.push_back(psbtx); // Choose the highest version number - if (static_cast<uint32_t>(psbtx.tx->nVersion) > best_version) { - best_version = static_cast<uint32_t>(psbtx.tx->nVersion); + if (psbtx.tx->version > best_version) { + best_version = psbtx.tx->version; } // Choose the lowest lock time if (psbtx.tx->nLockTime < best_locktime) { @@ -1756,7 +1760,7 @@ static RPCHelpMan joinpsbts() // Create a blank psbt where everything will be added PartiallySignedTransaction merged_psbt; merged_psbt.tx = CMutableTransaction(); - merged_psbt.tx->nVersion = static_cast<int32_t>(best_version); + merged_psbt.tx->version = best_version; merged_psbt.tx->nLockTime = best_locktime; // Merge @@ -1791,7 +1795,7 @@ static RPCHelpMan joinpsbts() PartiallySignedTransaction shuffled_psbt; shuffled_psbt.tx = CMutableTransaction(); - shuffled_psbt.tx->nVersion = merged_psbt.tx->nVersion; + shuffled_psbt.tx->version = merged_psbt.tx->version; shuffled_psbt.tx->nLockTime = merged_psbt.tx->nLockTime; for (int i : input_indices) { shuffled_psbt.AddInput(merged_psbt.tx->vin[i], merged_psbt.inputs[i]); @@ -1876,7 +1880,7 @@ static RPCHelpMan analyzepsbt() for (const CKeyID& pubkey : input.missing_pubkeys) { missing_pubkeys_univ.push_back(HexStr(pubkey)); } - missing.pushKV("pubkeys", missing_pubkeys_univ); + missing.pushKV("pubkeys", std::move(missing_pubkeys_univ)); } if (!input.missing_redeem_script.IsNull()) { missing.pushKV("redeemscript", HexStr(input.missing_redeem_script)); @@ -1889,14 +1893,14 @@ static RPCHelpMan analyzepsbt() for (const CKeyID& pubkey : input.missing_sigs) { missing_sigs_univ.push_back(HexStr(pubkey)); } - missing.pushKV("signatures", missing_sigs_univ); + missing.pushKV("signatures", std::move(missing_sigs_univ)); } if (!missing.getKeys().empty()) { - input_univ.pushKV("missing", missing); + input_univ.pushKV("missing", std::move(missing)); } - inputs_result.push_back(input_univ); + inputs_result.push_back(std::move(input_univ)); } - if (!inputs_result.empty()) result.pushKV("inputs", inputs_result); + if (!inputs_result.empty()) result.pushKV("inputs", std::move(inputs_result)); if (psbta.estimated_vsize != std::nullopt) { result.pushKV("estimated_vsize", (int)*psbta.estimated_vsize); diff --git a/src/rpc/rawtransaction_util.cpp b/src/rpc/rawtransaction_util.cpp index a9e11622a7..53f943bb9e 100644 --- a/src/rpc/rawtransaction_util.cpp +++ b/src/rpc/rawtransaction_util.cpp @@ -174,14 +174,14 @@ static void TxInErrorToJSON(const CTxIn& txin, UniValue& vErrorsRet, const std:: for (unsigned int i = 0; i < txin.scriptWitness.stack.size(); i++) { witness.push_back(HexStr(txin.scriptWitness.stack[i])); } - entry.pushKV("witness", witness); + entry.pushKV("witness", std::move(witness)); entry.pushKV("scriptSig", HexStr(txin.scriptSig)); entry.pushKV("sequence", (uint64_t)txin.nSequence); entry.pushKV("error", strMessage); - vErrorsRet.push_back(entry); + vErrorsRet.push_back(std::move(entry)); } -void ParsePrevouts(const UniValue& prevTxsUnival, FillableSigningProvider* keystore, std::map<COutPoint, Coin>& coins) +void ParsePrevouts(const UniValue& prevTxsUnival, FlatSigningProvider* keystore, std::map<COutPoint, Coin>& coins) { if (!prevTxsUnival.isNull()) { const UniValue& prevTxs = prevTxsUnival.get_array(); @@ -247,11 +247,11 @@ void ParsePrevouts(const UniValue& prevTxsUnival, FillableSigningProvider* keyst // work from witnessScript when possible std::vector<unsigned char> scriptData(!ws.isNull() ? ParseHexV(ws, "witnessScript") : ParseHexV(rs, "redeemScript")); CScript script(scriptData.begin(), scriptData.end()); - keystore->AddCScript(script); + keystore->scripts.emplace(CScriptID(script), script); // Automatically also add the P2WSH wrapped version of the script (to deal with P2SH-P2WSH). // This is done for redeemScript only for compatibility, it is encouraged to use the explicit witnessScript field instead. CScript witness_output_script{GetScriptForDestination(WitnessV0ScriptHash(script))}; - keystore->AddCScript(witness_output_script); + keystore->scripts.emplace(CScriptID(witness_output_script), witness_output_script); if (!ws.isNull() && !rs.isNull()) { // if both witnessScript and redeemScript are provided, @@ -331,6 +331,6 @@ void SignTransactionResultToJSON(CMutableTransaction& mtx, bool complete, const if (result.exists("errors")) { vErrors.push_backV(result["errors"].getValues()); } - result.pushKV("errors", vErrors); + result.pushKV("errors", std::move(vErrors)); } } diff --git a/src/rpc/rawtransaction_util.h b/src/rpc/rawtransaction_util.h index 964d0b095b..40d6bbba87 100644 --- a/src/rpc/rawtransaction_util.h +++ b/src/rpc/rawtransaction_util.h @@ -12,7 +12,7 @@ #include <optional> struct bilingual_str; -class FillableSigningProvider; +struct FlatSigningProvider; class UniValue; struct CMutableTransaction; class Coin; @@ -38,7 +38,7 @@ void SignTransactionResultToJSON(CMutableTransaction& mtx, bool complete, const * @param keystore A pointer to the temporary keystore if there is one * @param coins Map of unspent outputs - coins in mempool and current chain UTXO set, may be extended by previous txns outputs after call */ -void ParsePrevouts(const UniValue& prevTxsUnival, FillableSigningProvider* keystore, std::map<COutPoint, Coin>& coins); +void ParsePrevouts(const UniValue& prevTxsUnival, FlatSigningProvider* keystore, std::map<COutPoint, Coin>& coins); /** Normalize univalue-represented inputs and add them to the transaction */ void AddInputs(CMutableTransaction& rawTx, const UniValue& inputs_in, bool rbf); diff --git a/src/rpc/request.cpp b/src/rpc/request.cpp index d35782189e..87b9f18b33 100644 --- a/src/rpc/request.cpp +++ b/src/rpc/request.cpp @@ -45,6 +45,7 @@ UniValue JSONRPCRequestObj(const std::string& strMethod, const UniValue& params, request.pushKV("method", strMethod); request.pushKV("params", params); request.pushKV("id", id); + request.pushKV("jsonrpc", "2.0"); return request; } diff --git a/src/rpc/request.h b/src/rpc/request.h index e47f90af86..9968426636 100644 --- a/src/rpc/request.h +++ b/src/rpc/request.h @@ -17,6 +17,7 @@ enum class JSONRPCVersion { V2 }; +/** JSON-RPC 2.0 request, only used in bitcoin-cli **/ UniValue JSONRPCRequestObj(const std::string& strMethod, const UniValue& params, const UniValue& id); UniValue JSONRPCReplyObj(UniValue result, UniValue error, std::optional<UniValue> id, JSONRPCVersion jsonrpc_version); UniValue JSONRPCError(int code, const std::string& message); diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 1ed406354a..19063fa5be 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -27,6 +27,8 @@ #include <mutex> #include <unordered_map> +using util::SplitString; + static GlobalMutex g_rpc_warmup_mutex; static std::atomic<bool> g_rpc_running{false}; static bool fRPCInWarmup GUARDED_BY(g_rpc_warmup_mutex) = true; @@ -241,15 +243,15 @@ static RPCHelpMan getrpcinfo() UniValue entry(UniValue::VOBJ); entry.pushKV("method", info.method); entry.pushKV("duration", int64_t{Ticks<std::chrono::microseconds>(SteadyClock::now() - info.start)}); - active_commands.push_back(entry); + active_commands.push_back(std::move(entry)); } UniValue result(UniValue::VOBJ); - result.pushKV("active_commands", active_commands); + result.pushKV("active_commands", std::move(active_commands)); const std::string path = LogInstance().m_file_path.utf8string(); UniValue log_path(UniValue::VSTR, path); - result.pushKV("logpath", log_path); + result.pushKV("logpath", std::move(log_path)); return result; } diff --git a/src/rpc/server_util.cpp b/src/rpc/server_util.cpp index efd4a43c28..0387cbb8e2 100644 --- a/src/rpc/server_util.cpp +++ b/src/rpc/server_util.cpp @@ -101,6 +101,14 @@ CConnman& EnsureConnman(const NodeContext& node) return *node.connman; } +interfaces::Mining& EnsureMining(const NodeContext& node) +{ + if (!node.mining) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Node miner not found"); + } + return *node.mining; +} + PeerManager& EnsurePeerman(const NodeContext& node) { if (!node.peerman) { diff --git a/src/rpc/server_util.h b/src/rpc/server_util.h index a4a53166b4..1e6fb7e6a6 100644 --- a/src/rpc/server_util.h +++ b/src/rpc/server_util.h @@ -18,6 +18,9 @@ class BanMan; namespace node { struct NodeContext; } // namespace node +namespace interfaces { +class Mining; +} // namespace interfaces node::NodeContext& EnsureAnyNodeContext(const std::any& context); CTxMemPool& EnsureMemPool(const node::NodeContext& node); @@ -31,6 +34,7 @@ ChainstateManager& EnsureAnyChainman(const std::any& context); CBlockPolicyEstimator& EnsureFeeEstimator(const node::NodeContext& node); CBlockPolicyEstimator& EnsureAnyFeeEstimator(const std::any& context); CConnman& EnsureConnman(const node::NodeContext& node); +interfaces::Mining& EnsureMining(const node::NodeContext& node); PeerManager& EnsurePeerman(const node::NodeContext& node); AddrMan& EnsureAddrman(const node::NodeContext& node); AddrMan& EnsureAnyAddrman(const std::any& context); diff --git a/src/rpc/signmessage.cpp b/src/rpc/signmessage.cpp index 9f3c24efcf..83462738c5 100644 --- a/src/rpc/signmessage.cpp +++ b/src/rpc/signmessage.cpp @@ -3,6 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/signmessage.h> #include <key.h> #include <key_io.h> #include <rpc/protocol.h> @@ -10,7 +11,6 @@ #include <rpc/server.h> #include <rpc/util.h> #include <univalue.h> -#include <util/message.h> #include <string> diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index f5a2e9eb63..4df4466c49 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -5,14 +5,17 @@ #include <config/bitcoin-config.h> // IWYU pragma: keep #include <clientversion.h> -#include <core_io.h> #include <common/args.h> +#include <common/messages.h> +#include <common/types.h> #include <consensus/amount.h> -#include <script/interpreter.h> +#include <core_io.h> #include <key_io.h> +#include <node/types.h> #include <outputtype.h> #include <rpc/util.h> #include <script/descriptor.h> +#include <script/interpreter.h> #include <script/signingprovider.h> #include <script/solver.h> #include <tinyformat.h> @@ -22,7 +25,6 @@ #include <util/strencodings.h> #include <util/string.h> #include <util/translation.h> -#include <warnings.h> #include <algorithm> #include <iterator> @@ -30,6 +32,14 @@ #include <tuple> #include <utility> +using common::PSBTError; +using common::PSBTErrorString; +using common::TransactionErrorString; +using node::TransactionError; +using util::Join; +using util::SplitString; +using util::TrimString; + const std::string UNIX_EPOCH_TIME = "UNIX epoch time"; const std::string EXAMPLE_ADDRESS[2] = {"bc1q09vm5lfy0j5reeulh4x5752q25uqqvz34hufdl", "bc1q02ad21edsxd23d32dfgqqsz4vv4nmtfzuklhy3"}; @@ -176,7 +186,7 @@ std::string HelpExampleCliNamed(const std::string& methodname, const RPCArgList& std::string HelpExampleRpc(const std::string& methodname, const std::string& args) { return "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", " - "\"method\": \"" + methodname + "\", \"params\": [" + args + "]}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"; + "\"method\": \"" + methodname + "\", \"params\": [" + args + "]}' -H 'content-type: application/json' http://127.0.0.1:8332/\n"; } std::string HelpExampleRpcNamed(const std::string& methodname, const RPCArgList& args) @@ -187,7 +197,7 @@ std::string HelpExampleRpcNamed(const std::string& methodname, const RPCArgList& } return "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", " - "\"method\": \"" + methodname + "\", \"params\": " + params.write() + "}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"; + "\"method\": \"" + methodname + "\", \"params\": " + params.write() + "}' -H 'content-type: application/json' http://127.0.0.1:8332/\n"; } // Converts a hex string to a public key if possible @@ -228,7 +238,7 @@ CPubKey AddrToPubKey(const FillableSigningProvider& keystore, const std::string& } // Creates a multisig address from a given list of public keys, number of signatures required, and the address type -CTxDestination AddAndGetMultisigDestination(const int required, const std::vector<CPubKey>& pubkeys, OutputType type, FillableSigningProvider& keystore, CScript& script_out) +CTxDestination AddAndGetMultisigDestination(const int required, const std::vector<CPubKey>& pubkeys, OutputType type, FlatSigningProvider& keystore, CScript& script_out) { // Gather public keys if (required < 1) { @@ -364,6 +374,18 @@ unsigned int ParseConfirmTarget(const UniValue& value, unsigned int max_target) return unsigned_target; } +RPCErrorCode RPCErrorFromPSBTError(PSBTError err) +{ + switch (err) { + case PSBTError::UNSUPPORTED: + return RPC_INVALID_PARAMETER; + case PSBTError::SIGHASH_MISMATCH: + return RPC_DESERIALIZATION_ERROR; + default: break; + } + return RPC_TRANSACTION_ERROR; +} + RPCErrorCode RPCErrorFromTransactionError(TransactionError terr) { switch (terr) { @@ -371,18 +393,16 @@ RPCErrorCode RPCErrorFromTransactionError(TransactionError terr) return RPC_TRANSACTION_REJECTED; case TransactionError::ALREADY_IN_CHAIN: return RPC_TRANSACTION_ALREADY_IN_CHAIN; - case TransactionError::P2P_DISABLED: - return RPC_CLIENT_P2P_DISABLED; - case TransactionError::INVALID_PSBT: - case TransactionError::PSBT_MISMATCH: - return RPC_INVALID_PARAMETER; - case TransactionError::SIGHASH_MISMATCH: - return RPC_DESERIALIZATION_ERROR; default: break; } return RPC_TRANSACTION_ERROR; } +UniValue JSONRPCPSBTError(PSBTError err) +{ + return JSONRPCError(RPCErrorFromPSBTError(err), PSBTErrorString(err).original); +} + UniValue JSONRPCTransactionError(TransactionError terr, const std::string& err_string) { if (err_string.length() > 0) { @@ -645,7 +665,7 @@ UniValue RPCHelpMan::HandleRequest(const JSONRPCRequest& request) const mismatch.setNull(); break; } - mismatch.push_back(match); + mismatch.push_back(std::move(match)); } if (!mismatch.isNull()) { std::string explain{ @@ -677,7 +697,7 @@ static const UniValue* DetailMaybeArg(CheckFn* check, const std::vector<RPCArg>& static void CheckRequiredOrDefault(const RPCArg& param) { - // Must use `Arg<Type>(i)` to get the argument or its default value. + // Must use `Arg<Type>(key)` to get the argument or its default value. const bool required{ std::holds_alternative<RPCArg::Optional>(param.m_fallback) && RPCArg::Optional::NO == std::get<RPCArg::Optional>(param.m_fallback), }; @@ -778,7 +798,7 @@ std::string RPCHelpMan::ToString() const if (arg.m_opts.hidden) break; // Any arg that follows is also hidden // Push named argument name and description - sections.m_sections.emplace_back(::ToString(i + 1) + ". " + arg.GetFirstName(), arg.ToDescriptionString(/*is_named_arg=*/true)); + sections.m_sections.emplace_back(util::ToString(i + 1) + ". " + arg.GetFirstName(), arg.ToDescriptionString(/*is_named_arg=*/true)); sections.m_max_pad = std::max(sections.m_max_pad, sections.m_sections.back().m_left.size()); // Recursively push nested args @@ -818,7 +838,7 @@ UniValue RPCHelpMan::GetArgMap() const map.push_back(arg_name); map.push_back(type == RPCArg::Type::STR || type == RPCArg::Type::STR_HEX); - arr.push_back(map); + arr.push_back(std::move(map)); }; for (int i{0}; i < int(m_args.size()); ++i) { @@ -1124,7 +1144,7 @@ UniValue RPCResult::MatchesType(const UniValue& result) const // If there are more results than documented, reuse the last doc_inner. const RPCResult& doc_inner{m_inner.at(std::min(m_inner.size() - 1, i))}; UniValue match{doc_inner.MatchesType(result.get_array()[i])}; - if (!match.isTrue()) errors.pushKV(strprintf("%d", i), match); + if (!match.isTrue()) errors.pushKV(strprintf("%d", i), std::move(match)); } if (errors.empty()) return true; // empty result array is valid return errors; @@ -1137,7 +1157,7 @@ UniValue RPCResult::MatchesType(const UniValue& result) const const RPCResult& doc_inner{m_inner.at(0)}; // Assume all types are the same, randomly pick the first for (size_t i{0}; i < result.get_obj().size(); ++i) { UniValue match{doc_inner.MatchesType(result.get_obj()[i])}; - if (!match.isTrue()) errors.pushKV(result.getKeys()[i], match); + if (!match.isTrue()) errors.pushKV(result.getKeys()[i], std::move(match)); } if (errors.empty()) return true; // empty result obj is valid return errors; @@ -1163,7 +1183,7 @@ UniValue RPCResult::MatchesType(const UniValue& result) const continue; } UniValue match{doc_entry.MatchesType(result_it->second)}; - if (!match.isTrue()) errors.pushKV(doc_entry.m_key_name, match); + if (!match.isTrue()) errors.pushKV(doc_entry.m_key_name, std::move(match)); } if (errors.empty()) return true; return errors; @@ -1361,17 +1381,3 @@ void PushWarnings(const std::vector<bilingual_str>& warnings, UniValue& obj) if (warnings.empty()) return; obj.pushKV("warnings", BilingualStringsToUniValue(warnings)); } - -UniValue GetNodeWarnings(bool use_deprecated) -{ - if (use_deprecated) { - const auto all_warnings{GetWarnings()}; - return all_warnings.empty() ? "" : all_warnings.back().original; - } - - UniValue warnings{UniValue::VARR}; - for (auto&& warning : GetWarnings()) { - warnings.push_back(std::move(warning.original)); - } - return warnings; -} diff --git a/src/rpc/util.h b/src/rpc/util.h index 0e4dcc27b5..23024376e0 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -34,9 +34,14 @@ class JSONRPCRequest; enum ServiceFlags : uint64_t; enum class OutputType; -enum class TransactionError; struct FlatSigningProvider; struct bilingual_str; +namespace common { +enum class PSBTError; +} // namespace common +namespace node { +enum class TransactionError; +} // namespace node static constexpr bool DEFAULT_RPC_DOC_CHECK{ #ifdef RPC_DOC_CHECK @@ -117,7 +122,7 @@ std::string HelpExampleRpcNamed(const std::string& methodname, const RPCArgList& CPubKey HexToPubKey(const std::string& hex_in); CPubKey AddrToPubKey(const FillableSigningProvider& keystore, const std::string& addr_in); -CTxDestination AddAndGetMultisigDestination(const int required, const std::vector<CPubKey>& pubkeys, OutputType type, FillableSigningProvider& keystore, CScript& script_out); +CTxDestination AddAndGetMultisigDestination(const int required, const std::vector<CPubKey>& pubkeys, OutputType type, FlatSigningProvider& keystore, CScript& script_out); UniValue DescribeAddress(const CTxDestination& dest); @@ -127,8 +132,9 @@ int ParseSighashString(const UniValue& sighash); //! Parse a confirm target option and raise an RPC error if it is invalid. unsigned int ParseConfirmTarget(const UniValue& value, unsigned int max_target); -RPCErrorCode RPCErrorFromTransactionError(TransactionError terr); -UniValue JSONRPCTransactionError(TransactionError terr, const std::string& err_string = ""); +RPCErrorCode RPCErrorFromTransactionError(node::TransactionError terr); +UniValue JSONRPCPSBTError(common::PSBTError err); +UniValue JSONRPCTransactionError(node::TransactionError terr, const std::string& err_string = ""); //! Parse a JSON range specified as int64, or [int64, int64] std::pair<int64_t, int64_t> ParseDescriptorRange(const UniValue& value); @@ -414,19 +420,16 @@ public: * argument isNull() and parses (from JSON) and returns the user-passed argument, * or the default value derived from the RPCArg documentation. * - * There are two overloads of this function: - * - Use Arg<Type>(size_t i) to get the argument (or the default value) by index. - * - Use Arg<Type>(const std::string& key) to get the argument (or the default value) by key. - * - * The Type passed to this helper must match the corresponding RPCArg::Type. + * The instantiation of this helper for type R must match the corresponding RPCArg::Type. * - * @return The value of the RPC argument (or the default value) cast to type Type. + * @return The value of the RPC argument (or the default value) cast to type R. * * @see MaybeArg for handling optional arguments without default values. */ template <typename R> - auto Arg(size_t i) const + auto Arg(std::string_view key) const { + auto i{GetParamIndex(key)}; // Return argument (required or with default value). if constexpr (std::is_integral_v<R> || std::is_floating_point_v<R>) { // Return numbers by value. @@ -436,11 +439,6 @@ public: return ArgValue<const R&>(i); } } - template<typename R> - auto Arg(std::string_view key) const - { - return Arg<R>(GetParamIndex(key)); - } /** * @brief Helper to get an optional request argument. * @@ -452,21 +450,18 @@ public: * argument isNull() and parses (from JSON) and returns the user-passed argument, * or a falsy value if no argument was passed. * - * There are two overloads of this function: - * - Use MaybeArg<Type>(size_t i) to get the optional argument by index. - * - Use MaybeArg<Type>(const std::string& key) to get the optional argument by key. + * The instantiation of this helper for type R must match the corresponding RPCArg::Type. * - * The Type passed to this helper must match the corresponding RPCArg::Type. + * @return For integral and floating-point types, a std::optional<R> is returned. + * For other types, a R* pointer to the argument is returned. If the + * argument is not provided, std::nullopt or a null pointer is returned. * - * @return For integral and floating-point types, a std::optional<Type> is returned. - * For other types, a Type* pointer to the argument is returned. If the - * argument is not provided, std::nullopt or a null pointer is returned. - * * @see Arg for handling arguments that are required or have a default value. */ template <typename R> - auto MaybeArg(size_t i) const + auto MaybeArg(std::string_view key) const { + auto i{GetParamIndex(key)}; // Return optional argument (without default). if constexpr (std::is_integral_v<R> || std::is_floating_point_v<R>) { // Return numbers by value, wrapped in optional. @@ -476,11 +471,6 @@ public: return ArgValue<const R*>(i); } } - template<typename R> - auto MaybeArg(std::string_view key) const - { - return MaybeArg<R>(GetParamIndex(key)); - } std::string ToString() const; /** Return the named args that need to be converted from string to another JSON type */ UniValue GetArgMap() const; @@ -513,6 +503,4 @@ private: void PushWarnings(const UniValue& warnings, UniValue& obj); void PushWarnings(const std::vector<bilingual_str>& warnings, UniValue& obj); -UniValue GetNodeWarnings(bool use_deprecated); - #endif // BITCOIN_RPC_UTIL_H diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index a11d4dcbd5..0987db194c 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -8,6 +8,7 @@ #include <key_io.h> #include <pubkey.h> #include <script/miniscript.h> +#include <script/parsing.h> #include <script/script.h> #include <script/signingprovider.h> #include <script/solver.h> @@ -17,7 +18,6 @@ #include <span.h> #include <util/bip32.h> #include <util/check.h> -#include <util/spanparsing.h> #include <util/strencodings.h> #include <util/vector.h> @@ -27,6 +27,8 @@ #include <string> #include <vector> +using util::Split; + namespace { //////////////////////////////////////////////////////////////////////////// @@ -1350,8 +1352,6 @@ enum class ParseScriptContext { /** Parse a public key that excludes origin information. */ std::unique_ptr<PubkeyProvider> ParsePubkeyInner(uint32_t key_exp_index, const Span<const char>& sp, ParseScriptContext ctx, FlatSigningProvider& out, bool& apostrophe, std::string& error) { - using namespace spanparsing; - bool permit_uncompressed = ctx == ParseScriptContext::TOP || ctx == ParseScriptContext::P2SH; auto split = Split(sp, '/'); std::string str(split[0].begin(), split[0].end()); @@ -1424,8 +1424,6 @@ std::unique_ptr<PubkeyProvider> ParsePubkeyInner(uint32_t key_exp_index, const S /** Parse a public key including origin information (if enabled). */ std::unique_ptr<PubkeyProvider> ParsePubkey(uint32_t key_exp_index, const Span<const char>& sp, ParseScriptContext ctx, FlatSigningProvider& out, std::string& error) { - using namespace spanparsing; - auto origin_split = Split(sp, ']'); if (origin_split.size() > 2) { error = "Multiple ']' characters found for a single pubkey"; @@ -1589,7 +1587,7 @@ struct KeyParser { // NOLINTNEXTLINE(misc-no-recursion) std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const char>& sp, ParseScriptContext ctx, FlatSigningProvider& out, std::string& error) { - using namespace spanparsing; + using namespace script; auto expr = Expr(sp); if (Func("pk", expr)) { @@ -2038,8 +2036,6 @@ std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptCo /** Check a descriptor checksum, and update desc to be the checksum-less part. */ bool CheckChecksum(Span<const char>& sp, bool require_checksum, std::string& error, std::string* out_checksum = nullptr) { - using namespace spanparsing; - auto check_split = Split(sp, '#'); if (check_split.size() > 2) { error = "Multiple '#' symbols"; diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index c969ce45f1..20a9830d0e 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1321,8 +1321,8 @@ public: /** Serialize txTo */ template<typename S> void Serialize(S &s) const { - // Serialize nVersion - ::Serialize(s, txTo.nVersion); + // Serialize version + ::Serialize(s, txTo.version); // Serialize vin unsigned int nInputs = fAnyoneCanPay ? 1 : txTo.vin.size(); ::WriteCompactSize(s, nInputs); @@ -1512,7 +1512,7 @@ bool SignatureHashSchnorr(uint256& hash_out, ScriptExecutionData& execdata, cons ss << hash_type; // Transaction level data - ss << tx_to.nVersion; + ss << tx_to.version; ss << tx_to.nLockTime; if (input_type != SIGHASH_ANYONECANPAY) { ss << cache.m_prevouts_single_hash; @@ -1594,7 +1594,7 @@ uint256 SignatureHash(const CScript& scriptCode, const T& txTo, unsigned int nIn HashWriter ss{}; // Version - ss << txTo.nVersion; + ss << txTo.version; // Input prevouts/nSequence (none/all, depending on flags) ss << hashPrevouts; ss << hashSequence; @@ -1743,7 +1743,7 @@ bool GenericTransactionSignatureChecker<T>::CheckSequence(const CScriptNum& nSeq // Fail if the transaction's version number is not set high // enough to trigger BIP 68 rules. - if (static_cast<uint32_t>(txTo->nVersion) < 2) + if (txTo->version < 2) return false; // Sequence numbers with their most significant bit set are not diff --git a/src/script/miniscript.h b/src/script/miniscript.h index 4880f32410..a269709e72 100644 --- a/src/script/miniscript.h +++ b/src/script/miniscript.h @@ -18,10 +18,10 @@ #include <policy/policy.h> #include <primitives/transaction.h> +#include <script/parsing.h> #include <script/script.h> #include <span.h> #include <util/check.h> -#include <util/spanparsing.h> #include <util/strencodings.h> #include <util/string.h> #include <util/vector.h> @@ -251,7 +251,7 @@ namespace internal { //! The maximum size of a witness item for a Miniscript under Tapscript context. (A BIP340 signature with a sighash type byte.) static constexpr uint32_t MAX_TAPMINISCRIPT_STACK_ELEM_SIZE{65}; -//! nVersion + nLockTime +//! version + nLockTime constexpr uint32_t TX_OVERHEAD{4 + 4}; //! prevout + nSequence + scriptSig constexpr uint32_t TXIN_BYTES_NO_WITNESS{36 + 4 + 1}; @@ -863,8 +863,8 @@ public: if (!key_str) return {}; return std::move(ret) + "pk_h(" + std::move(*key_str) + ")"; } - case Fragment::AFTER: return std::move(ret) + "after(" + ::ToString(node.k) + ")"; - case Fragment::OLDER: return std::move(ret) + "older(" + ::ToString(node.k) + ")"; + case Fragment::AFTER: return std::move(ret) + "after(" + util::ToString(node.k) + ")"; + case Fragment::OLDER: return std::move(ret) + "older(" + util::ToString(node.k) + ")"; case Fragment::HASH256: return std::move(ret) + "hash256(" + HexStr(node.data) + ")"; case Fragment::HASH160: return std::move(ret) + "hash160(" + HexStr(node.data) + ")"; case Fragment::SHA256: return std::move(ret) + "sha256(" + HexStr(node.data) + ")"; @@ -883,7 +883,7 @@ public: return std::move(ret) + "andor(" + std::move(subs[0]) + "," + std::move(subs[1]) + "," + std::move(subs[2]) + ")"; case Fragment::MULTI: { CHECK_NONFATAL(!is_tapscript); - auto str = std::move(ret) + "multi(" + ::ToString(node.k); + auto str = std::move(ret) + "multi(" + util::ToString(node.k); for (const auto& key : node.keys) { auto key_str = ctx.ToString(key); if (!key_str) return {}; @@ -893,7 +893,7 @@ public: } case Fragment::MULTI_A: { CHECK_NONFATAL(is_tapscript); - auto str = std::move(ret) + "multi_a(" + ::ToString(node.k); + auto str = std::move(ret) + "multi_a(" + util::ToString(node.k); for (const auto& key : node.keys) { auto key_str = ctx.ToString(key); if (!key_str) return {}; @@ -902,7 +902,7 @@ public: return std::move(str) + ")"; } case Fragment::THRESH: { - auto str = std::move(ret) + "thresh(" + ::ToString(node.k); + auto str = std::move(ret) + "thresh(" + util::ToString(node.k); for (auto& sub : subs) { str += "," + std::move(sub); } @@ -1764,7 +1764,7 @@ void BuildBack(const MiniscriptContext script_ctx, Fragment nt, std::vector<Node template<typename Key, typename Ctx> inline NodeRef<Key> Parse(Span<const char> in, const Ctx& ctx) { - using namespace spanparsing; + using namespace script; // Account for the minimum script size for all parsed fragments so far. It "borrows" 1 // script byte from all leaf nodes, counting it instead whenever a space for a recursive diff --git a/src/util/spanparsing.cpp b/src/script/parsing.cpp index c464fc2b87..3528ac9bfa 100644 --- a/src/util/spanparsing.cpp +++ b/src/script/parsing.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <util/spanparsing.h> +#include <script/parsing.h> #include <span.h> @@ -10,7 +10,7 @@ #include <cstddef> #include <string> -namespace spanparsing { +namespace script { bool Const(const std::string& str, Span<const char>& sp) { @@ -49,4 +49,4 @@ Span<const char> Expr(Span<const char>& sp) return ret; } -} // namespace spanparsing +} // namespace script diff --git a/src/script/parsing.h b/src/script/parsing.h new file mode 100644 index 0000000000..850faea041 --- /dev/null +++ b/src/script/parsing.h @@ -0,0 +1,40 @@ +// Copyright (c) 2018-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_SCRIPT_PARSING_H +#define BITCOIN_SCRIPT_PARSING_H + +#include <span.h> + +#include <string> + +namespace script { + +/** Parse a constant. + * + * If sp's initial part matches str, sp is updated to skip that part, and true is returned. + * Otherwise sp is unmodified and false is returned. + */ +bool Const(const std::string& str, Span<const char>& sp); + +/** Parse a function call. + * + * If sp's initial part matches str + "(", and sp ends with ")", sp is updated to be the + * section between the braces, and true is returned. Otherwise sp is unmodified and false + * is returned. + */ +bool Func(const std::string& str, Span<const char>& sp); + +/** Extract the expression that sp begins with. + * + * This function will return the initial part of sp, up to (but not including) the first + * comma or closing brace, skipping ones that are surrounded by braces. So for example, + * for "foo(bar(1),2),3" the initial part "foo(bar(1),2)" will be returned. sp will be + * updated to skip the initial part that is returned. + */ +Span<const char> Expr(Span<const char>& sp); + +} // namespace script + +#endif // BITCOIN_SCRIPT_PARSING_H diff --git a/src/script/script.cpp b/src/script/script.cpp index 80e8d26bcf..73ea336c4f 100644 --- a/src/script/script.cpp +++ b/src/script/script.cpp @@ -6,10 +6,10 @@ #include <script/script.h> #include <crypto/common.h> +#include <crypto/hex_base.h> #include <hash.h> #include <uint256.h> #include <util/hash_type.h> -#include <util/strencodings.h> #include <string> diff --git a/src/signet.cpp b/src/signet.cpp index ebf0de09d3..7c193a1d77 100644 --- a/src/signet.cpp +++ b/src/signet.cpp @@ -68,13 +68,13 @@ static uint256 ComputeModifiedMerkleRoot(const CMutableTransaction& cb, const CB std::optional<SignetTxs> SignetTxs::Create(const CBlock& block, const CScript& challenge) { CMutableTransaction tx_to_spend; - tx_to_spend.nVersion = 0; + tx_to_spend.version = 0; tx_to_spend.nLockTime = 0; tx_to_spend.vin.emplace_back(COutPoint(), CScript(OP_0), 0); tx_to_spend.vout.emplace_back(0, challenge); CMutableTransaction tx_spending; - tx_spending.nVersion = 0; + tx_spending.version = 0; tx_spending.nLockTime = 0; tx_spending.vin.emplace_back(COutPoint(), CScript(), 0); tx_spending.vout.emplace_back(0, CScript(OP_RETURN)); diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 9668a85484..e5d25637bd 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -22,6 +22,7 @@ using namespace std::literals; using node::NodeContext; +using util::ToString; static NetGroupManager EMPTY_NETGROUPMAN{std::vector<bool>()}; static const bool DETERMINISTIC{true}; diff --git a/src/test/argsman_tests.cpp b/src/test/argsman_tests.cpp index 340208a1c9..5f0318e8c4 100644 --- a/src/test/argsman_tests.cpp +++ b/src/test/argsman_tests.cpp @@ -20,6 +20,8 @@ #include <boost/test/unit_test.hpp> +using util::ToString; + BOOST_FIXTURE_TEST_SUITE(argsman_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(util_datadir) diff --git a/src/test/blockchain_tests.cpp b/src/test/blockchain_tests.cpp index be515a9eac..9b8f419290 100644 --- a/src/test/blockchain_tests.cpp +++ b/src/test/blockchain_tests.cpp @@ -11,6 +11,8 @@ #include <cstdlib> +using util::ToString; + /* Equality between doubles is imprecise. Comparison should be done * with a small threshold of tolerance, rather than exact equality. */ diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index d44d84af93..067a32d6a4 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -67,7 +67,8 @@ CBlock BuildChainTestingSetup::CreateBlock(const CBlockIndex* prev, const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey) { - std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler{m_node.chainman->ActiveChainstate(), m_node.mempool.get()}.CreateNewBlock(scriptPubKey); + BlockAssembler::Options options; + std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler{m_node.chainman->ActiveChainstate(), m_node.mempool.get(), options}.CreateNewBlock(scriptPubKey); CBlock& block = pblocktemplate->block; block.hashPrevBlock = prev->GetBlockHash(); block.nTime = prev->nTime + 1; diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp index efe0983698..9eb7acc3ca 100644 --- a/src/test/blockmanager_tests.cpp +++ b/src/test/blockmanager_tests.cpp @@ -27,7 +27,7 @@ BOOST_FIXTURE_TEST_SUITE(blockmanager_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos) { const auto params {CreateChainParams(ArgsManager{}, ChainType::MAIN)}; - KernelNotifications notifications{*Assert(m_node.shutdown), m_node.exit_status}; + KernelNotifications notifications{*Assert(m_node.shutdown), m_node.exit_status, *Assert(m_node.warnings)}; const BlockManager::Options blockman_opts{ .chainparams = *params, .blocks_dir = m_args.GetBlocksDirPath(), @@ -134,7 +134,7 @@ BOOST_FIXTURE_TEST_CASE(blockmanager_block_data_availability, TestChain100Setup) BOOST_AUTO_TEST_CASE(blockmanager_flush_block_file) { - KernelNotifications notifications{*Assert(m_node.shutdown), m_node.exit_status}; + KernelNotifications notifications{*Assert(m_node.shutdown), m_node.exit_status, *Assert(m_node.warnings)}; node::BlockManager::Options blockman_opts{ .chainparams = Params(), .blocks_dir = m_args.GetBlocksDirPath(), diff --git a/src/test/data/tx_invalid.json b/src/test/data/tx_invalid.json index a47bc8f366..486469ddef 100644 --- a/src/test/data/tx_invalid.json +++ b/src/test/data/tx_invalid.json @@ -246,7 +246,7 @@ [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0x7c17aff532f22beb54069942f9bf567a66133eaf EQUAL"]], "0200000001000100000000000000000000000000000000000000000000000000000000000000000000030251b2000000000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], -["Failure due to insufficient tx.nVersion (<2)"], +["Failure due to insufficient tx.version (<2)"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0 CHECKSEQUENCEVERIFY 1"]], "010000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "CHECKSEQUENCEVERIFY"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4194304 CHECKSEQUENCEVERIFY"]], diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index 723a1ceee3..167e4be288 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -12,6 +12,8 @@ #include <boost/test/unit_test.hpp> +using util::ToString; + // Test if a string consists entirely of null characters static bool is_null_key(const std::vector<unsigned char>& key) { bool isnull = true; diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 5e9ae78681..42db28daf5 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -141,7 +141,7 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) { NodeId id{0}; auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman, Params()); - auto peerLogic = PeerManager::make(*connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, {}); + auto peerLogic = PeerManager::make(*connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, *m_node.warnings, {}); constexpr int max_outbound_full_relay = MAX_OUTBOUND_FULL_RELAY_CONNECTIONS; CConnman::Options options; @@ -239,7 +239,7 @@ BOOST_AUTO_TEST_CASE(block_relay_only_eviction) { NodeId id{0}; auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman, Params()); - auto peerLogic = PeerManager::make(*connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, {}); + auto peerLogic = PeerManager::make(*connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, *m_node.warnings, {}); constexpr int max_outbound_block_relay{MAX_BLOCK_RELAY_ONLY_CONNECTIONS}; constexpr int64_t MINIMUM_CONNECT_TIME{30}; @@ -300,7 +300,7 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) auto banman = std::make_unique<BanMan>(m_args.GetDataDirBase() / "banlist", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman, Params()); - auto peerLogic = PeerManager::make(*connman, *m_node.addrman, banman.get(), *m_node.chainman, *m_node.mempool, {}); + auto peerLogic = PeerManager::make(*connman, *m_node.addrman, banman.get(), *m_node.chainman, *m_node.mempool, *m_node.warnings, {}); CNetAddr tor_netaddr; BOOST_REQUIRE( @@ -330,7 +330,7 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) peerLogic->InitializeNode(*nodes[0], NODE_NETWORK); nodes[0]->fSuccessfullyConnected = true; connman->AddTestNode(*nodes[0]); - peerLogic->UnitTestMisbehaving(nodes[0]->GetId(), DISCOURAGEMENT_THRESHOLD); // Should be discouraged + peerLogic->UnitTestMisbehaving(nodes[0]->GetId()); // Should be discouraged BOOST_CHECK(peerLogic->SendMessages(nodes[0])); BOOST_CHECK(banman->IsDiscouraged(addr[0])); @@ -350,7 +350,6 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) peerLogic->InitializeNode(*nodes[1], NODE_NETWORK); nodes[1]->fSuccessfullyConnected = true; connman->AddTestNode(*nodes[1]); - peerLogic->UnitTestMisbehaving(nodes[1]->GetId(), DISCOURAGEMENT_THRESHOLD - 1); BOOST_CHECK(peerLogic->SendMessages(nodes[1])); // [0] is still discouraged/disconnected. BOOST_CHECK(banman->IsDiscouraged(addr[0])); @@ -358,7 +357,7 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) // [1] is not discouraged/disconnected yet. BOOST_CHECK(!banman->IsDiscouraged(addr[1])); BOOST_CHECK(!nodes[1]->fDisconnect); - peerLogic->UnitTestMisbehaving(nodes[1]->GetId(), 1); // [1] reaches discouragement threshold + peerLogic->UnitTestMisbehaving(nodes[1]->GetId()); BOOST_CHECK(peerLogic->SendMessages(nodes[1])); // Expect both [0] and [1] to be discouraged/disconnected now. BOOST_CHECK(banman->IsDiscouraged(addr[0])); @@ -381,7 +380,7 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) peerLogic->InitializeNode(*nodes[2], NODE_NETWORK); nodes[2]->fSuccessfullyConnected = true; connman->AddTestNode(*nodes[2]); - peerLogic->UnitTestMisbehaving(nodes[2]->GetId(), DISCOURAGEMENT_THRESHOLD); + peerLogic->UnitTestMisbehaving(nodes[2]->GetId()); BOOST_CHECK(peerLogic->SendMessages(nodes[2])); BOOST_CHECK(banman->IsDiscouraged(addr[0])); BOOST_CHECK(banman->IsDiscouraged(addr[1])); @@ -402,7 +401,7 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) auto banman = std::make_unique<BanMan>(m_args.GetDataDirBase() / "banlist", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman, Params()); - auto peerLogic = PeerManager::make(*connman, *m_node.addrman, banman.get(), *m_node.chainman, *m_node.mempool, {}); + auto peerLogic = PeerManager::make(*connman, *m_node.addrman, banman.get(), *m_node.chainman, *m_node.mempool, *m_node.warnings, {}); banman->ClearBanned(); int64_t nStartTime = GetTime(); @@ -423,7 +422,7 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) peerLogic->InitializeNode(dummyNode, NODE_NETWORK); dummyNode.fSuccessfullyConnected = true; - peerLogic->UnitTestMisbehaving(dummyNode.GetId(), DISCOURAGEMENT_THRESHOLD); + peerLogic->UnitTestMisbehaving(dummyNode.GetId()); BOOST_CHECK(peerLogic->SendMessages(&dummyNode)); BOOST_CHECK(banman->IsDiscouraged(addr)); diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index c779bf6f73..e6821dd321 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -7,6 +7,7 @@ #include <script/sign.h> #include <test/util/setup_common.h> #include <util/strencodings.h> +#include <util/string.h> #include <boost/test/unit_test.hpp> @@ -14,6 +15,8 @@ #include <string> #include <vector> +using util::Split; + namespace { void CheckUnparsable(const std::string& prv, const std::string& pub, const std::string& expected_error) @@ -400,7 +403,6 @@ void CheckInferDescriptor(const std::string& script_hex, const std::string& expe provider.pubkeys.emplace(origin_pubkey.GetID(), origin_pubkey); if (!origin_str.empty()) { - using namespace spanparsing; KeyOriginInfo info; Span<const char> origin_sp{origin_str}; std::vector<Span<const char>> origin_split = Split(origin_sp, "/"); diff --git a/src/test/fuzz/base_encode_decode.cpp b/src/test/fuzz/base_encode_decode.cpp index d322416d34..0cc8cb5886 100644 --- a/src/test/fuzz/base_encode_decode.cpp +++ b/src/test/fuzz/base_encode_decode.cpp @@ -14,6 +14,9 @@ #include <string> #include <vector> +using util::TrimString; +using util::TrimStringView; + FUZZ_TARGET(base_encode_decode) { const std::string random_encoded_string(buffer.begin(), buffer.end()); diff --git a/src/test/fuzz/bitset.cpp b/src/test/fuzz/bitset.cpp new file mode 100644 index 0000000000..7684337729 --- /dev/null +++ b/src/test/fuzz/bitset.cpp @@ -0,0 +1,316 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <span.h> +#include <test/fuzz/util.h> +#include <test/util/xoroshiro128plusplus.h> +#include <util/bitset.h> + +#include <bitset> +#include <vector> + +namespace { + +/** Pop the first byte from a byte-span, and return it. */ +uint8_t ReadByte(FuzzBufferType& buffer) +{ + if (buffer.empty()) return 0; + uint8_t ret = buffer.front(); + buffer = buffer.subspan(1); + return ret; +} + +/** Perform a simulation fuzz test on BitSet type S. */ +template<typename S> +void TestType(FuzzBufferType buffer) +{ + /** This fuzz test's design is based on the assumption that the actual bits stored in the + * bitsets and their simulations do not matter for the purpose of detecting edge cases, thus + * these are taken from a deterministically-seeded RNG instead. To provide some level of + * variation however, pick the seed based on the buffer size and size of the chosen bitset. */ + XoRoShiRo128PlusPlus rng(buffer.size() + 0x10000 * S::Size()); + + using Sim = std::bitset<S::Size()>; + // Up to 4 real BitSets (initially 2). + std::vector<S> real(2); + // Up to 4 std::bitsets with the same corresponding contents. + std::vector<Sim> sim(2); + + /* Compare sim[idx] with real[idx], using all inspector operations. */ + auto compare_fn = [&](unsigned idx) { + /* iterators and operator[] */ + auto it = real[idx].begin(); + unsigned first = S::Size(); + unsigned last = S::Size(); + for (unsigned i = 0; i < S::Size(); ++i) { + bool match = (it != real[idx].end()) && *it == i; + assert(sim[idx][i] == real[idx][i]); + assert(match == real[idx][i]); + assert((it == real[idx].end()) != (it != real[idx].end())); + if (match) { + ++it; + if (first == S::Size()) first = i; + last = i; + } + } + assert(it == real[idx].end()); + assert(!(it != real[idx].end())); + /* Any / None */ + assert(sim[idx].any() == real[idx].Any()); + assert(sim[idx].none() == real[idx].None()); + /* First / Last */ + if (sim[idx].any()) { + assert(first == real[idx].First()); + assert(last == real[idx].Last()); + } + /* Count */ + assert(sim[idx].count() == real[idx].Count()); + }; + + LIMITED_WHILE(buffer.size() > 0, 1000) { + // Read one byte to determine which operation to execute on the BitSets. + int command = ReadByte(buffer) % 64; + // Read another byte that determines which bitsets will be involved. + unsigned args = ReadByte(buffer); + unsigned dest = ((args & 7) * sim.size()) >> 3; + unsigned src = (((args >> 3) & 7) * sim.size()) >> 3; + unsigned aux = (((args >> 6) & 3) * sim.size()) >> 2; + // Args are in range for non-empty sim, or sim is completely empty and will be grown + assert((sim.empty() && dest == 0 && src == 0 && aux == 0) || + (!sim.empty() && dest < sim.size() && src < sim.size() && aux < sim.size())); + + // Pick one operation based on value of command. Not all operations are always applicable. + // Loop through the applicable ones until command reaches 0 (which avoids the need to + // compute the number of applicable commands ahead of time). + while (true) { + if (dest < sim.size() && command-- == 0) { + /* Set() (true) */ + unsigned val = ReadByte(buffer) % S::Size(); + assert(sim[dest][val] == real[dest][val]); + sim[dest].set(val); + real[dest].Set(val); + break; + } else if (dest < sim.size() && command-- == 0) { + /* Reset() */ + unsigned val = ReadByte(buffer) % S::Size(); + assert(sim[dest][val] == real[dest][val]); + sim[dest].reset(val); + real[dest].Reset(val); + break; + } else if (dest < sim.size() && command-- == 0) { + /* Set() (conditional) */ + unsigned val = ReadByte(buffer) % S::Size(); + assert(sim[dest][val] == real[dest][val]); + sim[dest].set(val, args >> 7); + real[dest].Set(val, args >> 7); + break; + } else if (sim.size() < 4 && command-- == 0) { + /* Construct empty. */ + sim.resize(sim.size() + 1); + real.resize(real.size() + 1); + break; + } else if (sim.size() < 4 && command-- == 0) { + /* Construct singleton. */ + unsigned val = ReadByte(buffer) % S::Size(); + std::bitset<S::Size()> newset; + newset[val] = true; + sim.push_back(newset); + real.push_back(S::Singleton(val)); + break; + } else if (dest < sim.size() && command-- == 0) { + /* Make random. */ + compare_fn(dest); + sim[dest].reset(); + real[dest] = S{}; + for (unsigned i = 0; i < S::Size(); ++i) { + if (rng() & 1) { + sim[dest][i] = true; + real[dest].Set(i); + } + } + break; + } else if (dest < sim.size() && command-- == 0) { + /* Assign initializer list. */ + unsigned r1 = rng() % S::Size(); + unsigned r2 = rng() % S::Size(); + unsigned r3 = rng() % S::Size(); + compare_fn(dest); + sim[dest].reset(); + real[dest] = {r1, r2, r3}; + sim[dest].set(r1); + sim[dest].set(r2); + sim[dest].set(r3); + break; + } else if (!sim.empty() && command-- == 0) { + /* Destruct. */ + compare_fn(sim.size() - 1); + sim.pop_back(); + real.pop_back(); + break; + } else if (sim.size() < 4 && src < sim.size() && command-- == 0) { + /* Copy construct. */ + sim.emplace_back(sim[src]); + real.emplace_back(real[src]); + break; + } else if (src < sim.size() && dest < sim.size() && command-- == 0) { + /* Copy assign. */ + compare_fn(dest); + sim[dest] = sim[src]; + real[dest] = real[src]; + break; + } else if (src < sim.size() && dest < sim.size() && command-- == 0) { + /* swap() function. */ + swap(sim[dest], sim[src]); + swap(real[dest], real[src]); + break; + } else if (sim.size() < 4 && command-- == 0) { + /* Construct with initializer list. */ + unsigned r1 = rng() % S::Size(); + unsigned r2 = rng() % S::Size(); + sim.emplace_back(); + sim.back().set(r1); + sim.back().set(r2); + real.push_back(S{r1, r2}); + break; + } else if (dest < sim.size() && command-- == 0) { + /* Fill() + copy assign. */ + unsigned len = ReadByte(buffer) % S::Size(); + compare_fn(dest); + sim[dest].reset(); + for (unsigned i = 0; i < len; ++i) sim[dest][i] = true; + real[dest] = S::Fill(len); + break; + } else if (src < sim.size() && command-- == 0) { + /* Iterator copy based compare. */ + unsigned val = ReadByte(buffer) % S::Size(); + /* In a first loop, compare begin..end, and copy to it_copy at some point. */ + auto it = real[src].begin(), it_copy = it; + for (unsigned i = 0; i < S::Size(); ++i) { + if (i == val) it_copy = it; + bool match = (it != real[src].end()) && *it == i; + assert(match == sim[src][i]); + if (match) ++it; + } + assert(it == real[src].end()); + /* Then compare from the copied point again to end. */ + for (unsigned i = val; i < S::Size(); ++i) { + bool match = (it_copy != real[src].end()) && *it_copy == i; + assert(match == sim[src][i]); + if (match) ++it_copy; + } + assert(it_copy == real[src].end()); + break; + } else if (src < sim.size() && dest < sim.size() && command-- == 0) { + /* operator|= */ + compare_fn(dest); + sim[dest] |= sim[src]; + real[dest] |= real[src]; + break; + } else if (src < sim.size() && dest < sim.size() && command-- == 0) { + /* operator&= */ + compare_fn(dest); + sim[dest] &= sim[src]; + real[dest] &= real[src]; + break; + } else if (src < sim.size() && dest < sim.size() && command-- == 0) { + /* operator-= */ + compare_fn(dest); + sim[dest] &= ~sim[src]; + real[dest] -= real[src]; + break; + } else if (src < sim.size() && dest < sim.size() && command-- == 0) { + /* operator^= */ + compare_fn(dest); + sim[dest] ^= sim[src]; + real[dest] ^= real[src]; + break; + } else if (src < sim.size() && dest < sim.size() && aux < sim.size() && command-- == 0) { + /* operator| */ + compare_fn(dest); + sim[dest] = sim[src] | sim[aux]; + real[dest] = real[src] | real[aux]; + break; + } else if (src < sim.size() && dest < sim.size() && aux < sim.size() && command-- == 0) { + /* operator& */ + compare_fn(dest); + sim[dest] = sim[src] & sim[aux]; + real[dest] = real[src] & real[aux]; + break; + } else if (src < sim.size() && dest < sim.size() && aux < sim.size() && command-- == 0) { + /* operator- */ + compare_fn(dest); + sim[dest] = sim[src] & ~sim[aux]; + real[dest] = real[src] - real[aux]; + break; + } else if (src < sim.size() && dest < sim.size() && aux < sim.size() && command-- == 0) { + /* operator^ */ + compare_fn(dest); + sim[dest] = sim[src] ^ sim[aux]; + real[dest] = real[src] ^ real[aux]; + break; + } else if (src < sim.size() && aux < sim.size() && command-- == 0) { + /* IsSupersetOf() and IsSubsetOf() */ + bool is_superset = (sim[aux] & ~sim[src]).none(); + bool is_subset = (sim[src] & ~sim[aux]).none(); + assert(real[src].IsSupersetOf(real[aux]) == is_superset); + assert(real[src].IsSubsetOf(real[aux]) == is_subset); + assert(real[aux].IsSupersetOf(real[src]) == is_subset); + assert(real[aux].IsSubsetOf(real[src]) == is_superset); + break; + } else if (src < sim.size() && aux < sim.size() && command-- == 0) { + /* operator== and operator!= */ + assert((sim[src] == sim[aux]) == (real[src] == real[aux])); + assert((sim[src] != sim[aux]) == (real[src] != real[aux])); + break; + } else if (src < sim.size() && aux < sim.size() && command-- == 0) { + /* Overlaps() */ + assert((sim[src] & sim[aux]).any() == real[src].Overlaps(real[aux])); + assert((sim[src] & sim[aux]).any() == real[aux].Overlaps(real[src])); + break; + } + } + } + /* Fully compare the final state. */ + for (unsigned i = 0; i < sim.size(); ++i) { + compare_fn(i); + } +} + +} // namespace + +FUZZ_TARGET(bitset) +{ + unsigned typdat = ReadByte(buffer) % 8; + if (typdat == 0) { + /* 16 bits */ + TestType<bitset_detail::IntBitSet<uint16_t>>(buffer); + TestType<bitset_detail::MultiIntBitSet<uint16_t, 1>>(buffer); + } else if (typdat == 1) { + /* 32 bits */ + TestType<bitset_detail::MultiIntBitSet<uint16_t, 2>>(buffer); + TestType<bitset_detail::IntBitSet<uint32_t>>(buffer); + } else if (typdat == 2) { + /* 48 bits */ + TestType<bitset_detail::MultiIntBitSet<uint16_t, 3>>(buffer); + } else if (typdat == 3) { + /* 64 bits */ + TestType<bitset_detail::IntBitSet<uint64_t>>(buffer); + TestType<bitset_detail::MultiIntBitSet<uint64_t, 1>>(buffer); + TestType<bitset_detail::MultiIntBitSet<uint32_t, 2>>(buffer); + TestType<bitset_detail::MultiIntBitSet<uint16_t, 4>>(buffer); + } else if (typdat == 4) { + /* 96 bits */ + TestType<bitset_detail::MultiIntBitSet<uint32_t, 3>>(buffer); + } else if (typdat == 5) { + /* 128 bits */ + TestType<bitset_detail::MultiIntBitSet<uint64_t, 2>>(buffer); + TestType<bitset_detail::MultiIntBitSet<uint32_t, 4>>(buffer); + } else if (typdat == 6) { + /* 192 bits */ + TestType<bitset_detail::MultiIntBitSet<uint64_t, 3>>(buffer); + } else if (typdat == 7) { + /* 256 bits */ + TestType<bitset_detail::MultiIntBitSet<uint64_t, 4>>(buffer); + } +} diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index c9a3bc86ac..6623edcf99 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -316,7 +316,8 @@ FUZZ_TARGET_DESERIALIZE(blocktransactionsrequest_deserialize, { DeserializeFromFuzzingInput(buffer, btr); }) FUZZ_TARGET_DESERIALIZE(snapshotmetadata_deserialize, { - SnapshotMetadata snapshot_metadata; + auto msg_start = Params().MessageStart(); + SnapshotMetadata snapshot_metadata{msg_start}; DeserializeFromFuzzingInput(buffer, snapshot_metadata); }) FUZZ_TARGET_DESERIALIZE(uint160_deserialize, { diff --git a/src/test/fuzz/fees.cpp b/src/test/fuzz/fees.cpp index 38a8c6798e..5c760be13d 100644 --- a/src/test/fuzz/fees.cpp +++ b/src/test/fuzz/fees.cpp @@ -2,17 +2,19 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/messages.h> #include <consensus/amount.h> #include <policy/fees.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> -#include <util/fees.h> #include <cstdint> #include <string> #include <vector> +using common::StringForFeeReason; + FUZZ_TARGET(fees) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index f9915187bd..c1c9945a04 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -79,10 +79,31 @@ void FuzzFrameworkRegisterTarget(std::string_view name, TypeTestOneInput target, static std::string_view g_fuzz_target; static const TypeTestOneInput* g_test_one_input{nullptr}; + +#if defined(__clang__) && defined(__linux__) +extern "C" void __llvm_profile_reset_counters(void) __attribute__((weak)); +extern "C" void __gcov_reset(void) __attribute__((weak)); + +void ResetCoverageCounters() +{ + if (__llvm_profile_reset_counters) { + __llvm_profile_reset_counters(); + } + + if (__gcov_reset) { + __gcov_reset(); + } +} +#else +void ResetCoverageCounters() {} +#endif + + void initialize() { - // Terminate immediately if a fuzzing harness ever tries to create a TCP socket. - CreateSock = [](const sa_family_t&) -> std::unique_ptr<Sock> { std::terminate(); }; + // Terminate immediately if a fuzzing harness ever tries to create a socket. + // Individual tests can override this by pointing CreateSock to a mocked alternative. + CreateSock = [](int, int, int) -> std::unique_ptr<Sock> { std::terminate(); }; // Terminate immediately if a fuzzing harness ever tries to perform a DNS lookup. g_dns_lookup = [](const std::string& name, bool allow_lookup) { @@ -129,6 +150,8 @@ void initialize() Assert(!g_test_one_input); g_test_one_input = &it->second.test_one_input; it->second.opts.init(); + + ResetCoverageCounters(); } #if defined(PROVIDE_FUZZ_MAIN_FUNCTION) diff --git a/src/test/fuzz/fuzz.h b/src/test/fuzz/fuzz.h index ca74d53de7..c468cd39e3 100644 --- a/src/test/fuzz/fuzz.h +++ b/src/test/fuzz/fuzz.h @@ -5,10 +5,9 @@ #ifndef BITCOIN_TEST_FUZZ_FUZZ_H #define BITCOIN_TEST_FUZZ_FUZZ_H -#include <span.h> - #include <cstdint> #include <functional> +#include <span> #include <string_view> /** @@ -23,7 +22,7 @@ #define LIMITED_WHILE(condition, limit) \ for (unsigned _count{limit}; (condition) && _count; --_count) -using FuzzBufferType = Span<const uint8_t>; +using FuzzBufferType = std::span<const uint8_t>; using TypeTestOneInput = std::function<void(FuzzBufferType)>; struct FuzzTargetOptions { diff --git a/src/test/fuzz/i2p.cpp b/src/test/fuzz/i2p.cpp new file mode 100644 index 0000000000..51517187a0 --- /dev/null +++ b/src/test/fuzz/i2p.cpp @@ -0,0 +1,63 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <common/args.h> +#include <i2p.h> +#include <netaddress.h> +#include <netbase.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/fuzz/util/net.h> +#include <test/util/setup_common.h> +#include <util/fs_helpers.h> +#include <util/threadinterrupt.h> + +void initialize_i2p() +{ + static const auto testing_setup = MakeNoLogFileContext<>(); +} + +FUZZ_TARGET(i2p, .init = initialize_i2p) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + + SetMockTime(ConsumeTime(fuzzed_data_provider)); + + // Mock CreateSock() to create FuzzedSock. + auto CreateSockOrig = CreateSock; + CreateSock = [&fuzzed_data_provider](int, int, int) { + return std::make_unique<FuzzedSock>(fuzzed_data_provider); + }; + + const fs::path private_key_path = gArgs.GetDataDirNet() / "fuzzed_i2p_private_key"; + const CService addr{in6_addr(IN6ADDR_LOOPBACK_INIT), 7656}; + const Proxy sam_proxy{addr, false}; + CThreadInterrupt interrupt; + + i2p::sam::Session session{private_key_path, sam_proxy, &interrupt}; + i2p::Connection conn; + + if (session.Listen(conn)) { + if (session.Accept(conn)) { + try { + (void)conn.sock->RecvUntilTerminator('\n', 10ms, interrupt, i2p::sam::MAX_MSG_SIZE); + } catch (const std::runtime_error&) { + } + } + } + + bool proxy_error; + + if (session.Connect(CService{}, conn, proxy_error)) { + try { + conn.sock->SendComplete("verack\n", 10ms, interrupt); + } catch (const std::runtime_error&) { + } + } + + fs::remove_all(private_key_path); + + CreateSock = CreateSockOrig; +} diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index db246bb84e..8f1d7b6d45 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -40,6 +40,8 @@ #include <set> #include <vector> +using util::ToString; + void initialize_integer() { SelectParams(ChainType::REGTEST); diff --git a/src/test/fuzz/key.cpp b/src/test/fuzz/key.cpp index d389a29575..82973803f8 100644 --- a/src/test/fuzz/key.cpp +++ b/src/test/fuzz/key.cpp @@ -78,16 +78,6 @@ FUZZ_TARGET(key, .init = initialize_key) assert(copied_key == key); } - { - CKey negated_key = key; - negated_key.Negate(); - assert(negated_key.IsValid()); - assert(!(negated_key == key)); - - negated_key.Negate(); - assert(negated_key == key); - } - const uint256 random_uint256 = Hash(buffer); { diff --git a/src/test/fuzz/kitchen_sink.cpp b/src/test/fuzz/kitchen_sink.cpp index 82f3a306c5..4468f358d9 100644 --- a/src/test/fuzz/kitchen_sink.cpp +++ b/src/test/fuzz/kitchen_sink.cpp @@ -2,13 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/messages.h> #include <merkleblock.h> +#include <node/types.h> #include <policy/fees.h> #include <rpc/util.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> -#include <util/error.h> #include <util/translation.h> #include <array> @@ -16,17 +17,15 @@ #include <optional> #include <vector> +using common::TransactionErrorString; +using node::TransactionError; + namespace { constexpr TransactionError ALL_TRANSACTION_ERROR[] = { - TransactionError::OK, TransactionError::MISSING_INPUTS, TransactionError::ALREADY_IN_CHAIN, - TransactionError::P2P_DISABLED, TransactionError::MEMPOOL_REJECTED, TransactionError::MEMPOOL_ERROR, - TransactionError::INVALID_PSBT, - TransactionError::PSBT_MISMATCH, - TransactionError::SIGHASH_MISMATCH, TransactionError::MAX_FEE_EXCEEDED, }; }; // namespace diff --git a/src/test/fuzz/locale.cpp b/src/test/fuzz/locale.cpp index 0f2985b504..68db842247 100644 --- a/src/test/fuzz/locale.cpp +++ b/src/test/fuzz/locale.cpp @@ -51,7 +51,7 @@ FUZZ_TARGET(locale) int64_t parseint64_out_without_locale; const bool parseint64_without_locale = ParseInt64(random_string, &parseint64_out_without_locale); const int64_t random_int64 = fuzzed_data_provider.ConsumeIntegral<int64_t>(); - const std::string tostring_without_locale = ToString(random_int64); + const std::string tostring_without_locale = util::ToString(random_int64); // The variable `random_int32` is no longer used, but the harness still needs to // consume the same data that it did previously to not invalidate existing seeds. const int32_t random_int32 = fuzzed_data_provider.ConsumeIntegral<int32_t>(); @@ -75,7 +75,7 @@ FUZZ_TARGET(locale) if (parseint64_without_locale) { assert(parseint64_out_without_locale == parseint64_out_with_locale); } - const std::string tostring_with_locale = ToString(random_int64); + const std::string tostring_with_locale = util::ToString(random_int64); assert(tostring_without_locale == tostring_with_locale); const std::string strprintf_int_with_locale = strprintf("%d", random_int64); assert(strprintf_int_without_locale == strprintf_int_with_locale); diff --git a/src/test/fuzz/message.cpp b/src/test/fuzz/message.cpp index 75baaa2754..6763206f72 100644 --- a/src/test/fuzz/message.cpp +++ b/src/test/fuzz/message.cpp @@ -3,12 +3,12 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> +#include <common/signmessage.h> #include <key_io.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <util/chaintype.h> -#include <util/message.h> #include <util/strencodings.h> #include <cassert> diff --git a/src/test/fuzz/mini_miner.cpp b/src/test/fuzz/mini_miner.cpp index 84f9bb4ad0..3a1663364f 100644 --- a/src/test/fuzz/mini_miner.cpp +++ b/src/test/fuzz/mini_miner.cpp @@ -7,11 +7,13 @@ #include <test/util/txmempool.h> #include <test/util/mining.h> -#include <node/mini_miner.h> #include <node/miner.h> +#include <node/mini_miner.h> #include <primitives/transaction.h> #include <random.h> #include <txmempool.h> +#include <util/check.h> +#include <util/translation.h> #include <deque> #include <vector> @@ -33,7 +35,9 @@ void initialize_miner() FUZZ_TARGET(mini_miner, .init = initialize_miner) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - CTxMemPool pool{CTxMemPool::Options{}}; + bilingual_str error; + CTxMemPool pool{CTxMemPool::Options{}, error}; + Assert(error.empty()); std::vector<COutPoint> outpoints; std::deque<COutPoint> available_coins = g_available_coins; LOCK2(::cs_main, pool.cs); @@ -109,7 +113,9 @@ FUZZ_TARGET(mini_miner, .init = initialize_miner) FUZZ_TARGET(mini_miner_selection, .init = initialize_miner) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - CTxMemPool pool{CTxMemPool::Options{}}; + bilingual_str error; + CTxMemPool pool{CTxMemPool::Options{}, error}; + Assert(error.empty()); // Make a copy to preserve determinism. std::deque<COutPoint> available_coins = g_available_coins; std::vector<CTransactionRef> transactions; diff --git a/src/test/fuzz/muhash.cpp b/src/test/fuzz/muhash.cpp index 8304e6fdb8..dd34c465ed 100644 --- a/src/test/fuzz/muhash.cpp +++ b/src/test/fuzz/muhash.cpp @@ -43,7 +43,19 @@ FUZZ_TARGET(muhash) }, [&] { // Test that dividing a MuHash by itself brings it back to it's initial state + + // See note about clang + self-assignment in test/uint256_tests.cpp + #if defined(__clang__) + # pragma clang diagnostic push + # pragma clang diagnostic ignored "-Wself-assign-overloaded" + #endif + muhash /= muhash; + + #if defined(__clang__) + # pragma clang diagnostic pop + #endif + muhash.Finalize(out); out2 = uint256S(initial_state_hash); }, diff --git a/src/test/fuzz/package_eval.cpp b/src/test/fuzz/package_eval.cpp index c201118bce..53aedf23ea 100644 --- a/src/test/fuzz/package_eval.cpp +++ b/src/test/fuzz/package_eval.cpp @@ -15,7 +15,9 @@ #include <test/util/script.h> #include <test/util/setup_common.h> #include <test/util/txmempool.h> +#include <util/check.h> #include <util/rbf.h> +#include <util/translation.h> #include <validation.h> #include <validationinterface.h> @@ -107,7 +109,7 @@ void MockTime(FuzzedDataProvider& fuzzed_data_provider, const Chainstate& chains SetMockTime(time); } -CTxMemPool MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeContext& node) +std::unique_ptr<CTxMemPool> MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeContext& node) { // Take the default options for tests... CTxMemPool::Options mempool_opts{MemPoolOptionsForTest(node)}; @@ -126,8 +128,13 @@ CTxMemPool MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeConte mempool_opts.check_ratio = 1; mempool_opts.require_standard = fuzzed_data_provider.ConsumeBool(); + bilingual_str error; // ...and construct a CTxMemPool from it - return CTxMemPool{mempool_opts}; + auto mempool{std::make_unique<CTxMemPool>(std::move(mempool_opts), error)}; + // ... ignore the error since it might be beneficial to fuzz even when the + // mempool size is unreasonably small + Assert(error.empty() || error.original.starts_with("-maxmempool must be at least ")); + return mempool; } FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) @@ -149,8 +156,8 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) auto outpoints_updater = std::make_shared<OutpointsUpdater>(mempool_outpoints); node.validation_signals->RegisterSharedValidationInterface(outpoints_updater); - CTxMemPool tx_pool_{MakeMempool(fuzzed_data_provider, node)}; - MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(&tx_pool_); + auto tx_pool_{MakeMempool(fuzzed_data_provider, node)}; + MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(tx_pool_.get()); chainstate.SetMempool(&tx_pool); @@ -173,7 +180,7 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) // Create transaction to add to the mempool const CTransactionRef tx = [&] { CMutableTransaction tx_mut; - tx_mut.nVersion = fuzzed_data_provider.ConsumeBool() ? 3 : CTransaction::CURRENT_VERSION; + tx_mut.version = fuzzed_data_provider.ConsumeBool() ? TRUC_VERSION : CTransaction::CURRENT_VERSION; tx_mut.nLockTime = fuzzed_data_provider.ConsumeBool() ? 0 : fuzzed_data_provider.ConsumeIntegral<uint32_t>(); // Last tx will sweep all outpoints in package const auto num_in = last_tx ? package_outpoints.size() : fuzzed_data_provider.ConsumeIntegralInRange<int>(1, mempool_outpoints.size()); @@ -307,7 +314,7 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) // just use result_package.m_state here. This makes the expect_valid check meaningless, but // we can still verify that the contents of m_tx_results are consistent with m_state. const bool expect_valid{result_package.m_state.IsValid()}; - Assert(!CheckPackageMempoolAcceptResult(txs, result_package, expect_valid, nullptr)); + Assert(!CheckPackageMempoolAcceptResult(txs, result_package, expect_valid, &tx_pool)); } else { // This is empty if it fails early checks, or "full" if transactions are looked at deeper Assert(result_package.m_tx_results.size() == txs.size() || result_package.m_tx_results.empty()); diff --git a/src/test/fuzz/partially_downloaded_block.cpp b/src/test/fuzz/partially_downloaded_block.cpp index 2bf47930f4..791d457710 100644 --- a/src/test/fuzz/partially_downloaded_block.cpp +++ b/src/test/fuzz/partially_downloaded_block.cpp @@ -10,6 +10,8 @@ #include <test/util/setup_common.h> #include <test/util/txmempool.h> #include <txmempool.h> +#include <util/check.h> +#include <util/translation.h> #include <cstddef> #include <cstdint> @@ -52,7 +54,9 @@ FUZZ_TARGET(partially_downloaded_block, .init = initialize_pdb) CBlockHeaderAndShortTxIDs cmpctblock{*block}; - CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; + bilingual_str error; + CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node), error}; + Assert(error.empty()); PartiallyDownloadedBlock pdb{&pool}; // Set of available transactions (mempool or extra_txn) diff --git a/src/test/fuzz/rbf.cpp b/src/test/fuzz/rbf.cpp index 64785948f6..eb981352ec 100644 --- a/src/test/fuzz/rbf.cpp +++ b/src/test/fuzz/rbf.cpp @@ -13,6 +13,8 @@ #include <test/util/setup_common.h> #include <test/util/txmempool.h> #include <txmempool.h> +#include <util/check.h> +#include <util/translation.h> #include <cstdint> #include <optional> @@ -56,7 +58,9 @@ FUZZ_TARGET(rbf, .init = initialize_rbf) return; } - CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; + bilingual_str error; + CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node), error}; + Assert(error.empty()); LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), NUM_ITERS) { @@ -87,10 +91,14 @@ FUZZ_TARGET(package_rbf, .init = initialize_package_rbf) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); SetMockTime(ConsumeTime(fuzzed_data_provider)); - std::optional<CMutableTransaction> child = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); - if (!child) return; + // "Real" virtual size is not important for this test since ConsumeTxMemPoolEntry generates its own virtual size values + // so we construct small transactions for performance reasons. Child simply needs an input for later to perhaps connect to parent. + CMutableTransaction child; + child.vin.resize(1); - CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; + bilingual_str error; + CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node), error}; + Assert(error.empty()); // Add a bunch of parent-child pairs to the mempool, and remember them. std::vector<CTransaction> mempool_txs; @@ -107,15 +115,13 @@ FUZZ_TARGET(package_rbf, .init = initialize_package_rbf) LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), NUM_ITERS) { // Make sure txns only have one input, and that a unique input is given to avoid circular references - std::optional<CMutableTransaction> parent = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); - if (!parent) { - return; - } + CMutableTransaction parent; assert(iter <= g_outpoints.size()); - parent->vin.resize(1); - parent->vin[0].prevout = g_outpoints[iter++]; + parent.vin.resize(1); + parent.vin[0].prevout = g_outpoints[iter++]; + parent.vout.emplace_back(0, CScript()); - mempool_txs.emplace_back(*parent); + mempool_txs.emplace_back(parent); const auto parent_entry = ConsumeTxMemPoolEntry(fuzzed_data_provider, mempool_txs.back()); running_vsize_total += parent_entry.GetTxSize(); if (running_vsize_total > std::numeric_limits<int32_t>::max()) { @@ -124,10 +130,10 @@ FUZZ_TARGET(package_rbf, .init = initialize_package_rbf) break; } pool.addUnchecked(parent_entry); - if (fuzzed_data_provider.ConsumeBool() && !child->vin.empty()) { - child->vin[0].prevout = COutPoint{mempool_txs.back().GetHash(), 0}; + if (fuzzed_data_provider.ConsumeBool()) { + child.vin[0].prevout = COutPoint{mempool_txs.back().GetHash(), 0}; } - mempool_txs.emplace_back(*child); + mempool_txs.emplace_back(child); const auto child_entry = ConsumeTxMemPoolEntry(fuzzed_data_provider, mempool_txs.back()); running_vsize_total += child_entry.GetTxSize(); if (running_vsize_total > std::numeric_limits<int32_t>::max()) { diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 2325bf0941..4e52c1c091 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -36,6 +36,9 @@ #include <vector> enum class ChainType; +using util::Join; +using util::ToString; + namespace { struct RPCFuzzTestingSetup : public TestingSetup { RPCFuzzTestingSetup(const ChainType chain_type, const std::vector<const char*>& extra_args) : TestingSetup{chain_type, extra_args} diff --git a/src/test/fuzz/script_assets_test_minimizer.cpp b/src/test/fuzz/script_assets_test_minimizer.cpp index 511b581f60..5a8b599df6 100644 --- a/src/test/fuzz/script_assets_test_minimizer.cpp +++ b/src/test/fuzz/script_assets_test_minimizer.cpp @@ -17,6 +17,8 @@ #include <string> #include <vector> +using util::SplitString; + // This fuzz "test" can be used to minimize test cases for script_assets_test in // src/test/script_tests.cpp. While it written as a fuzz test, and can be used as such, // fuzzing the inputs is unlikely to construct useful test cases. diff --git a/src/test/fuzz/spanparsing.cpp b/src/test/fuzz/script_parsing.cpp index b8996632bc..d29a6ea90c 100644 --- a/src/test/fuzz/spanparsing.cpp +++ b/src/test/fuzz/script_parsing.cpp @@ -2,11 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <script/parsing.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> -#include <util/spanparsing.h> +#include <util/string.h> -FUZZ_TARGET(spanparsing) +using util::Split; + +FUZZ_TARGET(script_parsing) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const size_t query_size = fuzzed_data_provider.ConsumeIntegral<size_t>(); @@ -15,16 +18,16 @@ FUZZ_TARGET(spanparsing) const Span<const char> const_span{span_str}; Span<const char> mut_span = const_span; - (void)spanparsing::Const(query, mut_span); + (void)script::Const(query, mut_span); mut_span = const_span; - (void)spanparsing::Func(query, mut_span); + (void)script::Func(query, mut_span); mut_span = const_span; - (void)spanparsing::Expr(mut_span); + (void)script::Expr(mut_span); if (!query.empty()) { mut_span = const_span; - (void)spanparsing::Split(mut_span, query.front()); + (void)Split(mut_span, query.front()); } } diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index 631da13803..5b822b03f6 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -5,6 +5,7 @@ #include <blockfilter.h> #include <clientversion.h> #include <common/args.h> +#include <common/messages.h> #include <common/settings.h> #include <common/system.h> #include <common/url.h> @@ -21,8 +22,6 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> -#include <util/error.h> -#include <util/fees.h> #include <util/strencodings.h> #include <util/string.h> #include <util/translation.h> @@ -37,6 +36,16 @@ enum class FeeEstimateMode; +using common::AmountErrMsg; +using common::AmountHighWarn; +using common::FeeModeFromString; +using common::ResolveErrMsg; +using util::ContainsNoNUL; +using util::Join; +using util::RemovePrefix; +using util::SplitString; +using util::TrimString; + FUZZ_TARGET(string) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); diff --git a/src/test/fuzz/timeoffsets.cpp b/src/test/fuzz/timeoffsets.cpp index 019337a94a..dfa4dd705d 100644 --- a/src/test/fuzz/timeoffsets.cpp +++ b/src/test/fuzz/timeoffsets.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <node/timeoffsets.h> +#include <node/warnings.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/util/setup_common.h> @@ -19,7 +20,8 @@ void initialize_timeoffsets() FUZZ_TARGET(timeoffsets, .init = initialize_timeoffsets) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - TimeOffsets offsets{}; + node::Warnings warnings{}; + TimeOffsets offsets{warnings}; LIMITED_WHILE(fuzzed_data_provider.remaining_bytes() > 0, 4'000) { (void)offsets.Median(); offsets.Add(std::chrono::seconds{fuzzed_data_provider.ConsumeIntegral<std::chrono::seconds::rep>()}); diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index 9f0aedf29b..b6b91445f9 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -15,12 +15,15 @@ #include <test/util/script.h> #include <test/util/setup_common.h> #include <test/util/txmempool.h> +#include <util/check.h> #include <util/rbf.h> +#include <util/translation.h> #include <validation.h> #include <validationinterface.h> using node::BlockAssembler; using node::NodeContext; +using util::ToString; namespace { @@ -116,7 +119,7 @@ void MockTime(FuzzedDataProvider& fuzzed_data_provider, const Chainstate& chains SetMockTime(time); } -CTxMemPool MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeContext& node) +std::unique_ptr<CTxMemPool> MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeContext& node) { // Take the default options for tests... CTxMemPool::Options mempool_opts{MemPoolOptionsForTest(node)}; @@ -126,7 +129,12 @@ CTxMemPool MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeConte mempool_opts.require_standard = fuzzed_data_provider.ConsumeBool(); // ...and construct a CTxMemPool from it - return CTxMemPool{mempool_opts}; + bilingual_str error; + auto mempool{std::make_unique<CTxMemPool>(std::move(mempool_opts), error)}; + // ... ignore the error since it might be beneficial to fuzz even when the + // mempool size is unreasonably small + Assert(error.empty() || error.original.starts_with("-maxmempool must be at least ")); + return mempool; } void CheckATMPInvariants(const MempoolAcceptResult& res, bool txid_in_mempool, bool wtxid_in_mempool) @@ -198,8 +206,8 @@ FUZZ_TARGET(tx_pool_standard, .init = initialize_tx_pool) constexpr CAmount SUPPLY_TOTAL{COINBASE_MATURITY * 50 * COIN}; SetMempoolConstraints(*node.args, fuzzed_data_provider); - CTxMemPool tx_pool_{MakeMempool(fuzzed_data_provider, node)}; - MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(&tx_pool_); + auto tx_pool_{MakeMempool(fuzzed_data_provider, node)}; + MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(tx_pool_.get()); chainstate.SetMempool(&tx_pool); @@ -226,7 +234,7 @@ FUZZ_TARGET(tx_pool_standard, .init = initialize_tx_pool) // Create transaction to add to the mempool const CTransactionRef tx = [&] { CMutableTransaction tx_mut; - tx_mut.nVersion = fuzzed_data_provider.ConsumeBool() ? 3 : CTransaction::CURRENT_VERSION; + tx_mut.version = fuzzed_data_provider.ConsumeBool() ? TRUC_VERSION : CTransaction::CURRENT_VERSION; tx_mut.nLockTime = fuzzed_data_provider.ConsumeBool() ? 0 : fuzzed_data_provider.ConsumeIntegral<uint32_t>(); const auto num_in = fuzzed_data_provider.ConsumeIntegralInRange<int>(1, outpoints_rbf.size()); const auto num_out = fuzzed_data_provider.ConsumeIntegralInRange<int>(1, outpoints_rbf.size() * 2); @@ -376,8 +384,8 @@ FUZZ_TARGET(tx_pool, .init = initialize_tx_pool) } SetMempoolConstraints(*node.args, fuzzed_data_provider); - CTxMemPool tx_pool_{MakeMempool(fuzzed_data_provider, node)}; - MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(&tx_pool_); + auto tx_pool_{MakeMempool(fuzzed_data_provider, node)}; + MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(tx_pool_.get()); chainstate.SetMempool(&tx_pool); diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index 259b00fcae..92ded99917 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -43,9 +43,9 @@ CMutableTransaction ConsumeTransaction(FuzzedDataProvider& fuzzed_data_provider, { CMutableTransaction tx_mut; const auto p2wsh_op_true = fuzzed_data_provider.ConsumeBool(); - tx_mut.nVersion = fuzzed_data_provider.ConsumeBool() ? + tx_mut.version = fuzzed_data_provider.ConsumeBool() ? CTransaction::CURRENT_VERSION : - fuzzed_data_provider.ConsumeIntegral<int32_t>(); + fuzzed_data_provider.ConsumeIntegral<uint32_t>(); tx_mut.nLockTime = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); const auto num_in = fuzzed_data_provider.ConsumeIntegralInRange<int>(0, max_num_in); const auto num_out = fuzzed_data_provider.ConsumeIntegralInRange<int>(0, max_num_out); diff --git a/src/test/fuzz/util/net.cpp b/src/test/fuzz/util/net.cpp index 99151bb84d..5e7aae670e 100644 --- a/src/test/fuzz/util/net.cpp +++ b/src/test/fuzz/util/net.cpp @@ -193,21 +193,20 @@ ssize_t FuzzedSock::Recv(void* buf, size_t len, int flags) const bool pad_to_len_bytes{m_fuzzed_data_provider.ConsumeBool()}; if (m_peek_data.has_value()) { // `MSG_PEEK` was used in the preceding `Recv()` call, return `m_peek_data`. - random_bytes.assign({m_peek_data.value()}); + random_bytes = m_peek_data.value(); if ((flags & MSG_PEEK) == 0) { m_peek_data.reset(); } pad_to_len_bytes = false; } else if ((flags & MSG_PEEK) != 0) { // New call with `MSG_PEEK`. - random_bytes = m_fuzzed_data_provider.ConsumeBytes<uint8_t>(1); + random_bytes = ConsumeRandomLengthByteVector(m_fuzzed_data_provider, len); if (!random_bytes.empty()) { - m_peek_data = random_bytes[0]; + m_peek_data = random_bytes; pad_to_len_bytes = false; } } else { - random_bytes = m_fuzzed_data_provider.ConsumeBytes<uint8_t>( - m_fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, len)); + random_bytes = ConsumeRandomLengthByteVector(m_fuzzed_data_provider, len); } if (random_bytes.empty()) { const ssize_t r = m_fuzzed_data_provider.ConsumeBool() ? 0 : -1; @@ -216,7 +215,11 @@ ssize_t FuzzedSock::Recv(void* buf, size_t len, int flags) const } return r; } - std::memcpy(buf, random_bytes.data(), random_bytes.size()); + // `random_bytes` might exceed the size of `buf` if e.g. Recv is called with + // len=N and MSG_PEEK first and afterwards called with len=M (M < N) and + // without MSG_PEEK. + size_t recv_len{std::min(random_bytes.size(), len)}; + std::memcpy(buf, random_bytes.data(), recv_len); if (pad_to_len_bytes) { if (len > random_bytes.size()) { std::memset((char*)buf + random_bytes.size(), 0, len - random_bytes.size()); @@ -226,7 +229,7 @@ ssize_t FuzzedSock::Recv(void* buf, size_t len, int flags) const if (m_fuzzed_data_provider.ConsumeBool() && std::getenv("FUZZED_SOCKET_FAKE_LATENCY") != nullptr) { std::this_thread::sleep_for(std::chrono::milliseconds{2}); } - return random_bytes.size(); + return recv_len; } int FuzzedSock::Connect(const sockaddr*, socklen_t) const @@ -380,7 +383,10 @@ bool FuzzedSock::Wait(std::chrono::milliseconds timeout, Event requested, Event* return false; } if (occurred != nullptr) { - *occurred = m_fuzzed_data_provider.ConsumeBool() ? requested : 0; + // We simulate the requested event as occured when ConsumeBool() + // returns false. This avoids simulating endless waiting if the + // FuzzedDataProvider runs out of data. + *occurred = m_fuzzed_data_provider.ConsumeBool() ? 0 : requested; } return true; } @@ -389,7 +395,10 @@ bool FuzzedSock::WaitMany(std::chrono::milliseconds timeout, EventsPerSock& even { for (auto& [sock, events] : events_per_sock) { (void)sock; - events.occurred = m_fuzzed_data_provider.ConsumeBool() ? events.requested : 0; + // We simulate the requested event as occured when ConsumeBool() + // returns false. This avoids simulating endless waiting if the + // FuzzedDataProvider runs out of data. + events.occurred = m_fuzzed_data_provider.ConsumeBool() ? 0 : events.requested; } return true; } diff --git a/src/test/fuzz/util/net.h b/src/test/fuzz/util/net.h index a6c9e23f2e..ed02680676 100644 --- a/src/test/fuzz/util/net.h +++ b/src/test/fuzz/util/net.h @@ -43,7 +43,7 @@ class FuzzedSock : public Sock * If `MSG_PEEK` is used, then our `Recv()` returns some random data as usual, but on the next * `Recv()` call we must return the same data, thus we remember it here. */ - mutable std::optional<uint8_t> m_peek_data; + mutable std::optional<std::vector<uint8_t>> m_peek_data; /** * Whether to pretend that the socket is select(2)-able. This is randomly set in the diff --git a/src/test/fuzz/utxo_snapshot.cpp b/src/test/fuzz/utxo_snapshot.cpp index dce728d96b..8c9c67a91c 100644 --- a/src/test/fuzz/utxo_snapshot.cpp +++ b/src/test/fuzz/utxo_snapshot.cpp @@ -47,7 +47,8 @@ FUZZ_TARGET(utxo_snapshot, .init = initialize_chain) const auto ActivateFuzzedSnapshot{[&] { AutoFile infile{fsbridge::fopen(snapshot_path, "rb")}; - SnapshotMetadata metadata; + auto msg_start = Params().MessageStart(); + SnapshotMetadata metadata{msg_start}; try { infile >> metadata; } catch (const std::ios_base::failure&) { diff --git a/src/test/fuzz/validation_load_mempool.cpp b/src/test/fuzz/validation_load_mempool.cpp index 00678742c9..51140ae039 100644 --- a/src/test/fuzz/validation_load_mempool.cpp +++ b/src/test/fuzz/validation_load_mempool.cpp @@ -13,7 +13,9 @@ #include <test/util/setup_common.h> #include <test/util/txmempool.h> #include <txmempool.h> +#include <util/check.h> #include <util/time.h> +#include <util/translation.h> #include <validation.h> #include <cstdint> @@ -40,7 +42,9 @@ FUZZ_TARGET(validation_load_mempool, .init = initialize_validation_load_mempool) SetMockTime(ConsumeTime(fuzzed_data_provider)); FuzzedFileProvider fuzzed_file_provider{fuzzed_data_provider}; - CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; + bilingual_str error; + CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node), error}; + Assert(error.empty()); auto& chainstate{static_cast<DummyChainState&>(g_setup->m_node.chainman->ActiveChainstate())}; chainstate.SetMempool(&pool); diff --git a/src/test/fuzz/vecdeque.cpp b/src/test/fuzz/vecdeque.cpp new file mode 100644 index 0000000000..1d9a98931f --- /dev/null +++ b/src/test/fuzz/vecdeque.cpp @@ -0,0 +1,491 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <span.h> +#include <test/fuzz/util.h> +#include <test/util/xoroshiro128plusplus.h> +#include <util/vecdeque.h> + +#include <deque> +#include <stdint.h> + +namespace { + +/** The maximum number of simultaneous buffers kept by the test. */ +static constexpr size_t MAX_BUFFERS{3}; +/** How many elements are kept in a buffer at most. */ +static constexpr size_t MAX_BUFFER_SIZE{48}; +/** How many operations are performed at most on the buffers in one test. */ +static constexpr size_t MAX_OPERATIONS{1024}; + +/** Perform a simulation fuzz test on VecDeque type T. + * + * T must be constructible from a uint64_t seed, comparable to other T, copyable, and movable. + */ +template<typename T, bool CheckNoneLeft> +void TestType(Span<const uint8_t> buffer, uint64_t rng_tweak) +{ + FuzzedDataProvider provider(buffer.data(), buffer.size()); + // Local RNG, only used for the seeds to initialize T objects with. + XoRoShiRo128PlusPlus rng(provider.ConsumeIntegral<uint64_t>() ^ rng_tweak); + + // Real circular buffers. + std::vector<VecDeque<T>> real; + real.reserve(MAX_BUFFERS); + // Simulated circular buffers. + std::vector<std::deque<T>> sim; + sim.reserve(MAX_BUFFERS); + // Temporary object of type T. + std::optional<T> tmp; + + // Compare a real and a simulated buffer. + auto compare_fn = [](const VecDeque<T>& r, const std::deque<T>& s) { + assert(r.size() == s.size()); + assert(r.empty() == s.empty()); + assert(r.capacity() >= r.size()); + if (s.size() == 0) return; + assert(r.front() == s.front()); + assert(r.back() == s.back()); + for (size_t i = 0; i < s.size(); ++i) { + assert(r[i] == s[i]); + } + }; + + LIMITED_WHILE(provider.remaining_bytes(), MAX_OPERATIONS) { + int command = provider.ConsumeIntegral<uint8_t>() % 64; + unsigned idx = real.empty() ? 0 : provider.ConsumeIntegralInRange<unsigned>(0, real.size() - 1); + const size_t num_buffers = sim.size(); + // Pick one operation based on value of command. Not all operations are always applicable. + // Loop through the applicable ones until command reaches 0 (which avoids the need to + // compute the number of applicable commands ahead of time). + const bool non_empty{num_buffers != 0}; + const bool non_full{num_buffers < MAX_BUFFERS}; + const bool partially_full{non_empty && non_full}; + const bool multiple_exist{num_buffers > 1}; + const bool existing_buffer_non_full{non_empty && sim[idx].size() < MAX_BUFFER_SIZE}; + const bool existing_buffer_non_empty{non_empty && !sim[idx].empty()}; + assert(non_full || non_empty); + while (true) { + if (non_full && command-- == 0) { + /* Default construct. */ + real.emplace_back(); + sim.emplace_back(); + break; + } + if (non_empty && command-- == 0) { + /* resize() */ + compare_fn(real[idx], sim[idx]); + size_t new_size = provider.ConsumeIntegralInRange<size_t>(0, MAX_BUFFER_SIZE); + real[idx].resize(new_size); + sim[idx].resize(new_size); + assert(real[idx].size() == new_size); + break; + } + if (non_empty && command-- == 0) { + /* clear() */ + compare_fn(real[idx], sim[idx]); + real[idx].clear(); + sim[idx].clear(); + assert(real[idx].empty()); + break; + } + if (non_empty && command-- == 0) { + /* Copy construct default. */ + compare_fn(real[idx], sim[idx]); + real[idx] = VecDeque<T>(); + sim[idx].clear(); + assert(real[idx].size() == 0); + break; + } + if (non_empty && command-- == 0) { + /* Destruct. */ + compare_fn(real.back(), sim.back()); + real.pop_back(); + sim.pop_back(); + break; + } + if (partially_full && command-- == 0) { + /* Copy construct. */ + real.emplace_back(real[idx]); + sim.emplace_back(sim[idx]); + break; + } + if (partially_full && command-- == 0) { + /* Move construct. */ + VecDeque<T> copy(real[idx]); + real.emplace_back(std::move(copy)); + sim.emplace_back(sim[idx]); + break; + } + if (multiple_exist && command-- == 0) { + /* swap() */ + swap(real[idx], real[(idx + 1) % num_buffers]); + swap(sim[idx], sim[(idx + 1) % num_buffers]); + break; + } + if (multiple_exist && command-- == 0) { + /* Copy assign. */ + compare_fn(real[idx], sim[idx]); + real[idx] = real[(idx + 1) % num_buffers]; + sim[idx] = sim[(idx + 1) % num_buffers]; + break; + } + if (multiple_exist && command-- == 0) { + /* Move assign. */ + VecDeque<T> copy(real[(idx + 1) % num_buffers]); + compare_fn(real[idx], sim[idx]); + real[idx] = std::move(copy); + sim[idx] = sim[(idx + 1) % num_buffers]; + break; + } + if (non_empty && command-- == 0) { + /* Self swap() */ + swap(real[idx], real[idx]); + break; + } + if (non_empty && command-- == 0) { + /* Self-copy assign. */ + real[idx] = real[idx]; + break; + } + if (non_empty && command-- == 0) { + /* Self-move assign. */ + // Do not use std::move(real[idx]) here: -Wself-move correctly warns about that. + real[idx] = static_cast<VecDeque<T>&&>(real[idx]); + break; + } + if (non_empty && command-- == 0) { + /* reserve() */ + size_t res_size = provider.ConsumeIntegralInRange<size_t>(0, MAX_BUFFER_SIZE); + size_t old_cap = real[idx].capacity(); + size_t old_size = real[idx].size(); + real[idx].reserve(res_size); + assert(real[idx].size() == old_size); + assert(real[idx].capacity() == std::max(old_cap, res_size)); + break; + } + if (non_empty && command-- == 0) { + /* shrink_to_fit() */ + size_t old_size = real[idx].size(); + real[idx].shrink_to_fit(); + assert(real[idx].size() == old_size); + assert(real[idx].capacity() == old_size); + break; + } + if (existing_buffer_non_full && command-- == 0) { + /* push_back() (copying) */ + tmp = T(rng()); + size_t old_size = real[idx].size(); + size_t old_cap = real[idx].capacity(); + real[idx].push_back(*tmp); + sim[idx].push_back(*tmp); + assert(real[idx].size() == old_size + 1); + if (old_cap > old_size) { + assert(real[idx].capacity() == old_cap); + } else { + assert(real[idx].capacity() > old_cap); + assert(real[idx].capacity() <= 2 * (old_cap + 1)); + } + break; + } + if (existing_buffer_non_full && command-- == 0) { + /* push_back() (moving) */ + tmp = T(rng()); + size_t old_size = real[idx].size(); + size_t old_cap = real[idx].capacity(); + sim[idx].push_back(*tmp); + real[idx].push_back(std::move(*tmp)); + assert(real[idx].size() == old_size + 1); + if (old_cap > old_size) { + assert(real[idx].capacity() == old_cap); + } else { + assert(real[idx].capacity() > old_cap); + assert(real[idx].capacity() <= 2 * (old_cap + 1)); + } + break; + } + if (existing_buffer_non_full && command-- == 0) { + /* emplace_back() */ + uint64_t seed{rng()}; + size_t old_size = real[idx].size(); + size_t old_cap = real[idx].capacity(); + sim[idx].emplace_back(seed); + real[idx].emplace_back(seed); + assert(real[idx].size() == old_size + 1); + if (old_cap > old_size) { + assert(real[idx].capacity() == old_cap); + } else { + assert(real[idx].capacity() > old_cap); + assert(real[idx].capacity() <= 2 * (old_cap + 1)); + } + break; + } + if (existing_buffer_non_full && command-- == 0) { + /* push_front() (copying) */ + tmp = T(rng()); + size_t old_size = real[idx].size(); + size_t old_cap = real[idx].capacity(); + real[idx].push_front(*tmp); + sim[idx].push_front(*tmp); + assert(real[idx].size() == old_size + 1); + if (old_cap > old_size) { + assert(real[idx].capacity() == old_cap); + } else { + assert(real[idx].capacity() > old_cap); + assert(real[idx].capacity() <= 2 * (old_cap + 1)); + } + break; + } + if (existing_buffer_non_full && command-- == 0) { + /* push_front() (moving) */ + tmp = T(rng()); + size_t old_size = real[idx].size(); + size_t old_cap = real[idx].capacity(); + sim[idx].push_front(*tmp); + real[idx].push_front(std::move(*tmp)); + assert(real[idx].size() == old_size + 1); + if (old_cap > old_size) { + assert(real[idx].capacity() == old_cap); + } else { + assert(real[idx].capacity() > old_cap); + assert(real[idx].capacity() <= 2 * (old_cap + 1)); + } + break; + } + if (existing_buffer_non_full && command-- == 0) { + /* emplace_front() */ + uint64_t seed{rng()}; + size_t old_size = real[idx].size(); + size_t old_cap = real[idx].capacity(); + sim[idx].emplace_front(seed); + real[idx].emplace_front(seed); + assert(real[idx].size() == old_size + 1); + if (old_cap > old_size) { + assert(real[idx].capacity() == old_cap); + } else { + assert(real[idx].capacity() > old_cap); + assert(real[idx].capacity() <= 2 * (old_cap + 1)); + } + break; + } + if (existing_buffer_non_empty && command-- == 0) { + /* front() [modifying] */ + tmp = T(rng()); + size_t old_size = real[idx].size(); + assert(sim[idx].front() == real[idx].front()); + sim[idx].front() = *tmp; + real[idx].front() = std::move(*tmp); + assert(real[idx].size() == old_size); + break; + } + if (existing_buffer_non_empty && command-- == 0) { + /* back() [modifying] */ + tmp = T(rng()); + size_t old_size = real[idx].size(); + assert(sim[idx].back() == real[idx].back()); + sim[idx].back() = *tmp; + real[idx].back() = *tmp; + assert(real[idx].size() == old_size); + break; + } + if (existing_buffer_non_empty && command-- == 0) { + /* operator[] [modifying] */ + tmp = T(rng()); + size_t pos = provider.ConsumeIntegralInRange<size_t>(0, sim[idx].size() - 1); + size_t old_size = real[idx].size(); + assert(sim[idx][pos] == real[idx][pos]); + sim[idx][pos] = *tmp; + real[idx][pos] = std::move(*tmp); + assert(real[idx].size() == old_size); + break; + } + if (existing_buffer_non_empty && command-- == 0) { + /* pop_front() */ + assert(sim[idx].front() == real[idx].front()); + size_t old_size = real[idx].size(); + sim[idx].pop_front(); + real[idx].pop_front(); + assert(real[idx].size() == old_size - 1); + break; + } + if (existing_buffer_non_empty && command-- == 0) { + /* pop_back() */ + assert(sim[idx].back() == real[idx].back()); + size_t old_size = real[idx].size(); + sim[idx].pop_back(); + real[idx].pop_back(); + assert(real[idx].size() == old_size - 1); + break; + } + } + } + + /* Fully compare the final state. */ + for (unsigned i = 0; i < sim.size(); ++i) { + // Make sure const getters work. + const VecDeque<T>& realbuf = real[i]; + const std::deque<T>& simbuf = sim[i]; + compare_fn(realbuf, simbuf); + for (unsigned j = 0; j < sim.size(); ++j) { + assert((realbuf == real[j]) == (simbuf == sim[j])); + assert(((realbuf <=> real[j]) >= 0) == (simbuf >= sim[j])); + assert(((realbuf <=> real[j]) <= 0) == (simbuf <= sim[j])); + } + // Clear out the buffers so we can check below that no objects exist anymore. + sim[i].clear(); + real[i].clear(); + } + + if constexpr (CheckNoneLeft) { + tmp = std::nullopt; + T::CheckNoneExist(); + } +} + +/** Data structure with built-in tracking of all existing objects. */ +template<size_t Size> +class TrackedObj +{ + static_assert(Size > 0); + + /* Data type for map that actually stores the object data. + * + * The key is a pointer to the TrackedObj, the value is the uint64_t it was initialized with. + * Default-constructed and moved-from objects hold an std::nullopt. + */ + using track_map_type = std::map<const TrackedObj<Size>*, std::optional<uint64_t>>; + +private: + + /** Actual map. */ + static inline track_map_type g_tracker; + + /** Iterators into the tracker map for this object. + * + * This is an array of size Size, all holding the same value, to give the object configurable + * size. The value is g_tracker.end() if this object is not fully initialized. */ + typename track_map_type::iterator m_track_entry[Size]; + + void Check() const + { + auto it = g_tracker.find(this); + for (size_t i = 0; i < Size; ++i) { + assert(m_track_entry[i] == it); + } + } + + /** Create entry for this object in g_tracker and populate m_track_entry. */ + void Register() + { + auto [it, inserted] = g_tracker.emplace(this, std::nullopt); + assert(inserted); + for (size_t i = 0; i < Size; ++i) { + m_track_entry[i] = it; + } + } + + void Deregister() + { + Check(); + assert(m_track_entry[0] != g_tracker.end()); + g_tracker.erase(m_track_entry[0]); + for (size_t i = 0; i < Size; ++i) { + m_track_entry[i] = g_tracker.end(); + } + } + + /** Get value corresponding to this object in g_tracker. */ + std::optional<uint64_t>& Deref() + { + Check(); + assert(m_track_entry[0] != g_tracker.end()); + return m_track_entry[0]->second; + } + + /** Get value corresponding to this object in g_tracker. */ + const std::optional<uint64_t>& Deref() const + { + Check(); + assert(m_track_entry[0] != g_tracker.end()); + return m_track_entry[0]->second; + } + +public: + ~TrackedObj() { Deregister(); } + TrackedObj() { Register(); } + + TrackedObj(uint64_t value) + { + Register(); + Deref() = value; + } + + TrackedObj(const TrackedObj& other) + { + Register(); + Deref() = other.Deref(); + } + + TrackedObj(TrackedObj&& other) + { + Register(); + Deref() = other.Deref(); + other.Deref() = std::nullopt; + } + + TrackedObj& operator=(const TrackedObj& other) + { + if (this == &other) return *this; + Deref() = other.Deref(); + return *this; + } + + TrackedObj& operator=(TrackedObj&& other) + { + if (this == &other) return *this; + Deref() = other.Deref(); + other.Deref() = std::nullopt; + return *this; + } + + friend bool operator==(const TrackedObj& a, const TrackedObj& b) + { + return a.Deref() == b.Deref(); + } + + friend std::strong_ordering operator<=>(const TrackedObj& a, const TrackedObj& b) + { + // Libc++ 15 & 16 do not support std::optional<T>::operator<=> yet. See + // https://reviews.llvm.org/D146392. + if (!a.Deref().has_value() || !b.Deref().has_value()) { + return a.Deref().has_value() <=> b.Deref().has_value(); + } + return *a.Deref() <=> *b.Deref(); + } + + static void CheckNoneExist() + { + assert(g_tracker.empty()); + } +}; + +} // namespace + +FUZZ_TARGET(vecdeque) +{ + // Run the test with simple uints (which satisfy all the trivial properties). + static_assert(std::is_trivially_copyable_v<uint32_t>); + static_assert(std::is_trivially_destructible_v<uint64_t>); + TestType<uint8_t, false>(buffer, 1); + TestType<uint16_t, false>(buffer, 2); + TestType<uint32_t, false>(buffer, 3); + TestType<uint64_t, false>(buffer, 4); + + // Run the test with TrackedObjs (which do not). + static_assert(!std::is_trivially_copyable_v<TrackedObj<3>>); + static_assert(!std::is_trivially_destructible_v<TrackedObj<17>>); + TestType<TrackedObj<1>, true>(buffer, 5); + TestType<TrackedObj<3>, true>(buffer, 6); + TestType<TrackedObj<17>, true>(buffer, 7); +} diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index c73b675388..8734735fd5 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -16,6 +16,8 @@ #include <boost/test/unit_test.hpp> +using util::SplitString; + BOOST_FIXTURE_TEST_SUITE(getarg_tests, BasicTestingSetup) void ResetArgs(ArgsManager& local_args, const std::string& strArg) diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index f0d2b9ed72..51f1d4c840 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -124,9 +124,9 @@ BOOST_AUTO_TEST_CASE(siphash) HashWriter ss{}; CMutableTransaction tx; - // Note these tests were originally written with tx.nVersion=1 + // Note these tests were originally written with tx.version=1 // and the test would be affected by default tx version bumps if not fixed. - tx.nVersion = 1; + tx.version = 1; ss << TX_WITH_WITNESS(tx); BOOST_CHECK_EQUAL(SipHashUint256(1, 2, ss.GetHash()), 0x79751e980c2a0a35ULL); diff --git a/src/test/i2p_tests.cpp b/src/test/i2p_tests.cpp index d7249d88f4..0512c6134f 100644 --- a/src/test/i2p_tests.cpp +++ b/src/test/i2p_tests.cpp @@ -39,15 +39,14 @@ public: private: const BCLog::Level m_prev_log_level; - const std::function<std::unique_ptr<Sock>(const sa_family_t&)> m_create_sock_orig; + const decltype(CreateSock) m_create_sock_orig; }; BOOST_FIXTURE_TEST_SUITE(i2p_tests, EnvTestingSetup) BOOST_AUTO_TEST_CASE(unlimited_recv) { - // Mock CreateSock() to create MockSock. - CreateSock = [](const sa_family_t&) { + CreateSock = [](int, int, int) { return std::make_unique<StaticContentsSock>(std::string(i2p::sam::MAX_MSG_SIZE + 1, 'a')); }; @@ -69,7 +68,7 @@ BOOST_AUTO_TEST_CASE(unlimited_recv) BOOST_AUTO_TEST_CASE(listen_ok_accept_fail) { size_t num_sockets{0}; - CreateSock = [&num_sockets](const sa_family_t&) { + CreateSock = [&num_sockets](int, int, int) { // clang-format off ++num_sockets; // First socket is the control socket for creating the session. @@ -133,9 +132,7 @@ BOOST_AUTO_TEST_CASE(listen_ok_accept_fail) BOOST_AUTO_TEST_CASE(damaged_private_key) { - const auto CreateSockOrig = CreateSock; - - CreateSock = [](const sa_family_t&) { + CreateSock = [](int, int, int) { return std::make_unique<StaticContentsSock>("HELLO REPLY RESULT=OK VERSION=3.1\n" "SESSION STATUS RESULT=OK DESTINATION=\n"); }; @@ -172,8 +169,6 @@ BOOST_AUTO_TEST_CASE(damaged_private_key) BOOST_CHECK(!session.Connect(CService{}, conn, proxy_error)); } } - - CreateSock = CreateSockOrig; } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index aaf4ca4977..b897a0a153 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -19,6 +19,8 @@ #include <boost/test/unit_test.hpp> +using util::ToString; + static const std::string strSecret1 = "5HxWvvfubhXpYYpS3tJkw6fq9jE9j18THftkZjHHfmFiWtmAbrj"; static const std::string strSecret2 = "5KC4ejrDjv152FGwP386VD1i2NYc5KkfSMyv1nGy1VGDxGHqVY3"; static const std::string strSecret1C = "Kwr371tjA9u2rFSMZjTNun2PXXP3WPZu2afRHTcta6KxEUdm1vEw"; @@ -201,37 +203,6 @@ BOOST_AUTO_TEST_CASE(key_signature_tests) BOOST_CHECK(found_small); } -BOOST_AUTO_TEST_CASE(key_key_negation) -{ - // create a dummy hash for signature comparison - unsigned char rnd[8]; - std::string str = "Bitcoin key verification\n"; - GetRandBytes(rnd); - uint256 hash{Hash(str, rnd)}; - - // import the static test key - CKey key = DecodeSecret(strSecret1C); - - // create a signature - std::vector<unsigned char> vch_sig; - std::vector<unsigned char> vch_sig_cmp; - key.Sign(hash, vch_sig); - - // negate the key twice - BOOST_CHECK(key.GetPubKey().data()[0] == 0x03); - key.Negate(); - // after the first negation, the signature must be different - key.Sign(hash, vch_sig_cmp); - BOOST_CHECK(vch_sig_cmp != vch_sig); - BOOST_CHECK(key.GetPubKey().data()[0] == 0x02); - key.Negate(); - // after the second negation, we should have the original key and thus the - // same signature - key.Sign(hash, vch_sig_cmp); - BOOST_CHECK(vch_sig_cmp == vch_sig); - BOOST_CHECK(key.GetPubKey().data()[0] == 0x03); -} - static CPubKey UnserializePubkey(const std::vector<uint8_t>& data) { DataStream stream{}; diff --git a/src/test/logging_tests.cpp b/src/test/logging_tests.cpp index 88e3ec94b7..cebd000df8 100644 --- a/src/test/logging_tests.cpp +++ b/src/test/logging_tests.cpp @@ -17,6 +17,9 @@ #include <boost/test/unit_test.hpp> +using util::SplitString; +using util::TrimString; + BOOST_FIXTURE_TEST_SUITE(logging_tests, BasicTestingSetup) static void ResetLogger() diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index d50af4c175..c4cf6f8a40 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -14,8 +14,10 @@ #include <test/util/txmempool.h> #include <txmempool.h> #include <uint256.h> +#include <util/check.h> #include <util/strencodings.h> #include <util/time.h> +#include <util/translation.h> #include <validation.h> #include <versionbits.h> @@ -46,7 +48,9 @@ struct MinerTestingSetup : public TestingSetup { // pointer is not accessed, when the new one should be accessed // instead. m_node.mempool.reset(); - m_node.mempool = std::make_unique<CTxMemPool>(MemPoolOptionsForTest(m_node)); + bilingual_str error; + m_node.mempool = std::make_unique<CTxMemPool>(MemPoolOptionsForTest(m_node), error); + Assert(error.empty()); return *m_node.mempool; } BlockAssembler AssemblerForTest(CTxMemPool& tx_mempool); @@ -421,7 +425,7 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: std::vector<int> prevheights; // relative height locked - tx.nVersion = 2; + tx.version = 2; tx.vin.resize(1); prevheights.resize(1); tx.vin[0].prevout.hash = txFirst[0]->GetHash(); // only 1 transaction @@ -622,7 +626,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) pblock->nVersion = VERSIONBITS_TOP_BITS; pblock->nTime = m_node.chainman->ActiveChain().Tip()->GetMedianTimePast()+1; CMutableTransaction txCoinbase(*pblock->vtx[0]); - txCoinbase.nVersion = 1; + txCoinbase.version = 1; txCoinbase.vin[0].scriptSig = CScript{} << (m_node.chainman->ActiveChain().Height() + 1) << bi.extranonce; txCoinbase.vout.resize(1); // Ignore the (optional) segwit commitment added by CreateNewBlock (as the hardcoded nonces don't account for this) txCoinbase.vout[0].scriptPubKey = CScript(); diff --git a/src/test/net_peer_connection_tests.cpp b/src/test/net_peer_connection_tests.cpp index 00bc1fdb6a..5f38ce112c 100644 --- a/src/test/net_peer_connection_tests.cpp +++ b/src/test/net_peer_connection_tests.cpp @@ -84,7 +84,7 @@ static void AddPeer(NodeId& id, std::vector<CNode*>& nodes, PeerManager& peerman BOOST_AUTO_TEST_CASE(test_addnode_getaddednodeinfo_and_connection_detection) { auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman, Params()); - auto peerman = PeerManager::make(*connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, {}); + auto peerman = PeerManager::make(*connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, *m_node.warnings, {}); NodeId id{0}; std::vector<CNode*> nodes; diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index b9dff96610..46a6a33b34 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -32,6 +32,7 @@ #include <string> using namespace std::literals; +using util::ToString; BOOST_FIXTURE_TEST_SUITE(net_tests, RegTestingSetup) diff --git a/src/test/node_warnings_tests.cpp b/src/test/node_warnings_tests.cpp new file mode 100644 index 0000000000..2bcc2c95ed --- /dev/null +++ b/src/test/node_warnings_tests.cpp @@ -0,0 +1,52 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// + +#include <node/warnings.h> +#include <util/translation.h> + +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(node_warnings_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(warnings) +{ + node::Warnings warnings; + // On pre-release builds, a warning is generated automatically + warnings.Unset(node::Warning::PRE_RELEASE_TEST_BUILD); + + // For these tests, we don't care what the exact warnings are, so + // just refer to them as warning_1 and warning_2 + const auto warning_1{node::Warning::CLOCK_OUT_OF_SYNC}; + const auto warning_2{node::Warning::FATAL_INTERNAL_ERROR}; + + // Ensure we start without any warnings + BOOST_CHECK(warnings.GetMessages().size() == 0); + // Add two warnings + BOOST_CHECK(warnings.Set(warning_1, _("warning 1"))); + BOOST_CHECK(warnings.Set(warning_2, _("warning 2"))); + // Unset the second one + BOOST_CHECK(warnings.Unset(warning_2)); + // Since it's already been unset, this should return false + BOOST_CHECK(!warnings.Unset(warning_2)); + // We should now be able to set w2 again + BOOST_CHECK(warnings.Set(warning_2, _("warning 2 - revision 1"))); + // Setting w2 again should return false since it's already set + BOOST_CHECK(!warnings.Set(warning_2, _("warning 2 - revision 2"))); + + // Verify messages are correct + const auto messages{warnings.GetMessages()}; + BOOST_CHECK(messages.size() == 2); + BOOST_CHECK(messages[0].original == "warning 1"); + BOOST_CHECK(messages[1].original == "warning 2 - revision 1"); + + // Clearing all warnings should also clear all messages + BOOST_CHECK(warnings.Unset(warning_1)); + BOOST_CHECK(warnings.Unset(warning_2)); + BOOST_CHECK(warnings.GetMessages().size() == 0); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/peerman_tests.cpp b/src/test/peerman_tests.cpp index 28866695bc..6de373eef2 100644 --- a/src/test/peerman_tests.cpp +++ b/src/test/peerman_tests.cpp @@ -20,7 +20,8 @@ static void mineBlock(const node::NodeContext& node, std::chrono::seconds block_ { auto curr_time = GetTime<std::chrono::seconds>(); SetMockTime(block_time); // update time so the block is created with it - CBlock block = node::BlockAssembler{node.chainman->ActiveChainstate(), nullptr}.CreateNewBlock(CScript() << OP_TRUE)->block; + node::BlockAssembler::Options options; + CBlock block = node::BlockAssembler{node.chainman->ActiveChainstate(), nullptr, options}.CreateNewBlock(CScript() << OP_TRUE)->block; while (!CheckProofOfWork(block.GetHash(), block.nBits, node.chainman->GetConsensus())) ++block.nNonce; block.fChecked = true; // little speedup SetMockTime(curr_time); // process block at current time @@ -31,7 +32,7 @@ static void mineBlock(const node::NodeContext& node, std::chrono::seconds block_ // Verifying when network-limited peer connections are desirable based on the node's proximity to the tip BOOST_AUTO_TEST_CASE(connections_desirable_service_flags) { - std::unique_ptr<PeerManager> peerman = PeerManager::make(*m_node.connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, {}); + std::unique_ptr<PeerManager> peerman = PeerManager::make(*m_node.connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, *m_node.warnings, {}); auto consensus = m_node.chainman->GetParams().GetConsensus(); // Check we start connecting to full nodes diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index 1c7d11d8a4..5089f3e8e3 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -17,6 +17,8 @@ #include <boost/test/unit_test.hpp> +using util::SplitString; + static UniValue JSON(std::string_view json) { UniValue value; @@ -552,7 +554,7 @@ BOOST_AUTO_TEST_CASE(help_example) // test different argument types const RPCArgList& args = {{"foo", "bar"}, {"b", true}, {"n", 1}}; BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", args), "> bitcoin-cli -named test foo=bar b=true n=1\n"); - BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", args), "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"foo\":\"bar\",\"b\":true,\"n\":1}}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"); + BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", args), "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"foo\":\"bar\",\"b\":true,\"n\":1}}' -H 'content-type: application/json' http://127.0.0.1:8332/\n"); // test shell escape BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", {{"foo", "b'ar"}}), "> bitcoin-cli -named test foo='b'''ar'\n"); @@ -565,7 +567,7 @@ BOOST_AUTO_TEST_CASE(help_example) obj_value.pushKV("b", false); obj_value.pushKV("n", 1); BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", {{"name", obj_value}}), "> bitcoin-cli -named test name='{\"foo\":\"bar\",\"b\":false,\"n\":1}'\n"); - BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", {{"name", obj_value}}), "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"name\":{\"foo\":\"bar\",\"b\":false,\"n\":1}}}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"); + BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", {{"name", obj_value}}), "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"name\":{\"foo\":\"bar\",\"b\":false,\"n\":1}}}' -H 'content-type: application/json' http://127.0.0.1:8332/\n"); // test array params UniValue arr_value(UniValue::VARR); @@ -573,7 +575,7 @@ BOOST_AUTO_TEST_CASE(help_example) arr_value.push_back(false); arr_value.push_back(1); BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", {{"name", arr_value}}), "> bitcoin-cli -named test name='[\"bar\",false,1]'\n"); - BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", {{"name", arr_value}}), "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"name\":[\"bar\",false,1]}}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"); + BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", {{"name", arr_value}}), "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"name\":[\"bar\",false,1]}}' -H 'content-type: application/json' http://127.0.0.1:8332/\n"); // test types don't matter for shell BOOST_CHECK_EQUAL(HelpExampleCliNamed("foo", {{"arg", true}}), HelpExampleCliNamed("foo", {{"arg", "true"}})); @@ -614,40 +616,26 @@ BOOST_AUTO_TEST_CASE(rpc_arg_helper) //! Check that `self.Arg` returns the same value as the `request.params` accessors RPCHelpMan::RPCMethodImpl check_positional = [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - BOOST_CHECK_EQUAL(self.Arg<int>(0), request.params[0].getInt<int>()); - BOOST_CHECK_EQUAL(self.Arg<std::string>(1), request.params[1].get_str()); - BOOST_CHECK_EQUAL(self.Arg<uint64_t>(2), request.params[2].isNull() ? DEFAULT_UINT64_T : request.params[2].getInt<uint64_t>()); - BOOST_CHECK_EQUAL(self.Arg<std::string>(3), request.params[3].isNull() ? DEFAULT_STRING : request.params[3].get_str()); - BOOST_CHECK_EQUAL(self.Arg<bool>(4), request.params[4].isNull() ? DEFAULT_BOOL : request.params[4].get_bool()); + BOOST_CHECK_EQUAL(self.Arg<int>("req_int"), request.params[0].getInt<int>()); + BOOST_CHECK_EQUAL(self.Arg<std::string>("req_str"), request.params[1].get_str()); + BOOST_CHECK_EQUAL(self.Arg<uint64_t>("def_uint64_t"), request.params[2].isNull() ? DEFAULT_UINT64_T : request.params[2].getInt<uint64_t>()); + BOOST_CHECK_EQUAL(self.Arg<std::string>("def_string"), request.params[3].isNull() ? DEFAULT_STRING : request.params[3].get_str()); + BOOST_CHECK_EQUAL(self.Arg<bool>("def_bool"), request.params[4].isNull() ? DEFAULT_BOOL : request.params[4].get_bool()); if (!request.params[5].isNull()) { - BOOST_CHECK_EQUAL(self.MaybeArg<double>(5).value(), request.params[5].get_real()); + BOOST_CHECK_EQUAL(self.MaybeArg<double>("opt_double").value(), request.params[5].get_real()); } else { - BOOST_CHECK(!self.MaybeArg<double>(5)); + BOOST_CHECK(!self.MaybeArg<double>("opt_double")); } if (!request.params[6].isNull()) { - BOOST_CHECK(self.MaybeArg<std::string>(6)); - BOOST_CHECK_EQUAL(*self.MaybeArg<std::string>(6), request.params[6].get_str()); + BOOST_CHECK(self.MaybeArg<std::string>("opt_string")); + BOOST_CHECK_EQUAL(*self.MaybeArg<std::string>("opt_string"), request.params[6].get_str()); } else { - BOOST_CHECK(!self.MaybeArg<std::string>(6)); + BOOST_CHECK(!self.MaybeArg<std::string>("opt_string")); } return UniValue{}; }; CheckRpc(params, UniValue{JSON(R"([5, "hello", null, null, null, null, null])")}, check_positional); CheckRpc(params, UniValue{JSON(R"([5, "hello", 4, "test", true, 1.23, "world"])")}, check_positional); - - //! Check that `self.Arg` returns the same value when using index and key - RPCHelpMan::RPCMethodImpl check_named = [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - BOOST_CHECK_EQUAL(self.Arg<int>(0), self.Arg<int>("req_int")); - BOOST_CHECK_EQUAL(self.Arg<std::string>(1), self.Arg<std::string>("req_str")); - BOOST_CHECK_EQUAL(self.Arg<uint64_t>(2), self.Arg<uint64_t>("def_uint64_t")); - BOOST_CHECK_EQUAL(self.Arg<std::string>(3), self.Arg<std::string>("def_string")); - BOOST_CHECK_EQUAL(self.Arg<bool>(4), self.Arg<bool>("def_bool")); - BOOST_CHECK(self.MaybeArg<double>(5) == self.MaybeArg<double>("opt_double")); - BOOST_CHECK(self.MaybeArg<std::string>(6) == self.MaybeArg<std::string>("opt_string")); - return UniValue{}; - }; - CheckRpc(params, UniValue{JSON(R"([5, "hello", null, null, null, null, null])")}, check_named); - CheckRpc(params, UniValue{JSON(R"([5, "hello", 4, "test", true, 1.23, "world"])")}, check_named); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 314b26609c..39b53295e7 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -396,7 +396,7 @@ public: wit.push_back(HexStr(scriptWitness.stack[i])); } wit.push_back(ValueFromAmount(nValue)); - array.push_back(wit); + array.push_back(std::move(wit)); } array.push_back(FormatScript(spendTx.vin[0].scriptSig)); array.push_back(FormatScript(creditTx->vout[0].scriptPubKey)); diff --git a/src/test/settings_tests.cpp b/src/test/settings_tests.cpp index 41190b3579..95f38fc0ce 100644 --- a/src/test/settings_tests.cpp +++ b/src/test/settings_tests.cpp @@ -21,6 +21,8 @@ #include <system_error> #include <vector> +using util::ToString; + inline bool operator==(const common::SettingsValue& a, const common::SettingsValue& b) { return a.write() == b.write(); diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index d08d5519a1..70a18835d2 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -92,7 +92,7 @@ void static RandomScript(CScript &script) { void static RandomTransaction(CMutableTransaction& tx, bool fSingle) { - tx.nVersion = int(InsecureRand32()); + tx.version = InsecureRand32(); tx.vin.clear(); tx.vout.clear(); tx.nLockTime = (InsecureRandBool()) ? InsecureRand32() : 0; diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index 2081acdf4d..aed67d5f3c 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -86,7 +86,7 @@ static ScriptError VerifyWithFlag(const CTransaction& output, const CMutableTran */ static void BuildTxs(CMutableTransaction& spendingTx, CCoinsViewCache& coins, CMutableTransaction& creationTx, const CScript& scriptPubKey, const CScript& scriptSig, const CScriptWitness& witness) { - creationTx.nVersion = 1; + creationTx.version = 1; creationTx.vin.resize(1); creationTx.vin[0].prevout.SetNull(); creationTx.vin[0].scriptSig = CScript(); @@ -94,7 +94,7 @@ static void BuildTxs(CMutableTransaction& spendingTx, CCoinsViewCache& coins, CM creationTx.vout[0].nValue = 1; creationTx.vout[0].scriptPubKey = scriptPubKey; - spendingTx.nVersion = 1; + spendingTx.version = 1; spendingTx.vin.resize(1); spendingTx.vin[0].prevout.hash = creationTx.GetHash(); spendingTx.vin[0].prevout.n = 0; diff --git a/src/test/timeoffsets_tests.cpp b/src/test/timeoffsets_tests.cpp index 008f1a9db9..5f85a5feeb 100644 --- a/src/test/timeoffsets_tests.cpp +++ b/src/test/timeoffsets_tests.cpp @@ -4,6 +4,7 @@ // #include <node/timeoffsets.h> +#include <node/warnings.h> #include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> @@ -24,7 +25,8 @@ BOOST_FIXTURE_TEST_SUITE(timeoffsets_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(timeoffsets) { - TimeOffsets offsets{}; + node::Warnings warnings{}; + TimeOffsets offsets{warnings}; BOOST_CHECK(offsets.Median() == 0s); AddMulti(offsets, {{0s, -1s, -2s, -3s}}); @@ -50,7 +52,8 @@ BOOST_AUTO_TEST_CASE(timeoffsets) static bool IsWarningRaised(const std::vector<std::chrono::seconds>& check_offsets) { - TimeOffsets offsets{}; + node::Warnings warnings{}; + TimeOffsets offsets{warnings}; AddMulti(offsets, check_offsets); return offsets.WarnIfOutOfSync(); } diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index e6cf64611e..34176626f0 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -38,6 +38,9 @@ #include <univalue.h> +using util::SplitString; +using util::ToString; + typedef std::vector<unsigned char> valtype; static CFeeRate g_dust{DUST_RELAY_TX_FEE}; @@ -69,17 +72,16 @@ static std::map<std::string, unsigned int> mapFlagNames = { unsigned int ParseScriptFlags(std::string strFlags) { - if (strFlags.empty() || strFlags == "NONE") return 0; - unsigned int flags = 0; - std::vector<std::string> words = SplitString(strFlags, ','); + unsigned int flags = SCRIPT_VERIFY_NONE; + if (strFlags.empty() || strFlags == "NONE") return flags; + std::vector<std::string> words = SplitString(strFlags, ','); for (const std::string& word : words) { if (!mapFlagNames.count(word)) BOOST_ERROR("Bad test: unknown verification flag '" << word << "'"); flags |= mapFlagNames[word]; } - return flags; } @@ -95,7 +97,7 @@ bool CheckMapFlagNames() std::string FormatScriptFlags(unsigned int flags) { - if (flags == 0) { + if (flags == SCRIPT_VERIFY_NONE) { return ""; } std::string ret; @@ -367,6 +369,41 @@ BOOST_AUTO_TEST_CASE(tx_invalid) } } +BOOST_AUTO_TEST_CASE(tx_no_inputs) +{ + CMutableTransaction empty; + + TxValidationState state; + BOOST_CHECK_MESSAGE(!CheckTransaction(CTransaction(empty), state), "Transaction with no inputs should be invalid."); + BOOST_CHECK(state.GetRejectReason() == "bad-txns-vin-empty"); +} + +BOOST_AUTO_TEST_CASE(tx_oversized) +{ + auto createTransaction =[](size_t payloadSize) { + CMutableTransaction tx; + tx.vin.resize(1); + tx.vout.emplace_back(1, CScript() << OP_RETURN << std::vector<unsigned char>(payloadSize)); + return CTransaction(tx); + }; + const auto maxTransactionSize = MAX_BLOCK_WEIGHT / WITNESS_SCALE_FACTOR; + const auto oversizedTransactionBaseSize = ::GetSerializeSize(TX_NO_WITNESS(createTransaction(maxTransactionSize))) - maxTransactionSize; + + auto maxPayloadSize = maxTransactionSize - oversizedTransactionBaseSize; + { + TxValidationState state; + CheckTransaction(createTransaction(maxPayloadSize), state); + BOOST_CHECK(state.GetRejectReason() != "bad-txns-oversize"); + } + + maxPayloadSize += 1; + { + TxValidationState state; + BOOST_CHECK_MESSAGE(!CheckTransaction(createTransaction(maxPayloadSize), state), "Oversized transaction should be invalid"); + BOOST_CHECK(state.GetRejectReason() == "bad-txns-oversize"); + } +} + BOOST_AUTO_TEST_CASE(basic_transaction_tests) { // Random real transaction (e2769b09e784f32f62ef849763d4f45b98e07ba658647343b915ff832b110436) @@ -412,7 +449,7 @@ BOOST_AUTO_TEST_CASE(test_Get) static void CreateCreditAndSpend(const FillableSigningProvider& keystore, const CScript& outscript, CTransactionRef& output, CMutableTransaction& input, bool success = true) { CMutableTransaction outputm; - outputm.nVersion = 1; + outputm.version = 1; outputm.vin.resize(1); outputm.vin[0].prevout.SetNull(); outputm.vin[0].scriptSig = CScript(); @@ -428,7 +465,7 @@ static void CreateCreditAndSpend(const FillableSigningProvider& keystore, const assert(output->vout[0] == outputm.vout[0]); CMutableTransaction inputm; - inputm.nVersion = 1; + inputm.version = 1; inputm.vin.resize(1); inputm.vin[0].prevout.hash = output->GetHash(); inputm.vin[0].prevout.n = 0; @@ -485,7 +522,7 @@ static void ReplaceRedeemScript(CScript& script, const CScript& redeemScript) BOOST_AUTO_TEST_CASE(test_big_witness_transaction) { CMutableTransaction mtx; - mtx.nVersion = 1; + mtx.version = 1; CKey key = GenerateRandomKey(); // Need to use compressed keys in segwit or the signing will fail FillableSigningProvider keystore; @@ -612,11 +649,11 @@ BOOST_AUTO_TEST_CASE(test_witness) // Normal pay-to-compressed-pubkey. CreateCreditAndSpend(keystore, scriptPubkey1, output1, input1); CreateCreditAndSpend(keystore, scriptPubkey2, output2, input2); - CheckWithFlag(output1, input1, 0, true); + CheckWithFlag(output1, input1, SCRIPT_VERIFY_NONE, true); CheckWithFlag(output1, input1, SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output1, input1, SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output1, input1, STANDARD_SCRIPT_VERIFY_FLAGS, true); - CheckWithFlag(output1, input2, 0, false); + CheckWithFlag(output1, input2, SCRIPT_VERIFY_NONE, false); CheckWithFlag(output1, input2, SCRIPT_VERIFY_P2SH, false); CheckWithFlag(output1, input2, SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH, false); CheckWithFlag(output1, input2, STANDARD_SCRIPT_VERIFY_FLAGS, false); @@ -625,11 +662,11 @@ BOOST_AUTO_TEST_CASE(test_witness) CreateCreditAndSpend(keystore, GetScriptForDestination(ScriptHash(scriptPubkey1)), output1, input1); CreateCreditAndSpend(keystore, GetScriptForDestination(ScriptHash(scriptPubkey2)), output2, input2); ReplaceRedeemScript(input2.vin[0].scriptSig, scriptPubkey1); - CheckWithFlag(output1, input1, 0, true); + CheckWithFlag(output1, input1, SCRIPT_VERIFY_NONE, true); CheckWithFlag(output1, input1, SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output1, input1, SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output1, input1, STANDARD_SCRIPT_VERIFY_FLAGS, true); - CheckWithFlag(output1, input2, 0, true); + CheckWithFlag(output1, input2, SCRIPT_VERIFY_NONE, true); CheckWithFlag(output1, input2, SCRIPT_VERIFY_P2SH, false); CheckWithFlag(output1, input2, SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH, false); CheckWithFlag(output1, input2, STANDARD_SCRIPT_VERIFY_FLAGS, false); @@ -637,11 +674,11 @@ BOOST_AUTO_TEST_CASE(test_witness) // Witness pay-to-compressed-pubkey (v0). CreateCreditAndSpend(keystore, destination_script_1, output1, input1); CreateCreditAndSpend(keystore, destination_script_2, output2, input2); - CheckWithFlag(output1, input1, 0, true); + CheckWithFlag(output1, input1, SCRIPT_VERIFY_NONE, true); CheckWithFlag(output1, input1, SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output1, input1, SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output1, input1, STANDARD_SCRIPT_VERIFY_FLAGS, true); - CheckWithFlag(output1, input2, 0, true); + CheckWithFlag(output1, input2, SCRIPT_VERIFY_NONE, true); CheckWithFlag(output1, input2, SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output1, input2, SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH, false); CheckWithFlag(output1, input2, STANDARD_SCRIPT_VERIFY_FLAGS, false); @@ -650,11 +687,11 @@ BOOST_AUTO_TEST_CASE(test_witness) CreateCreditAndSpend(keystore, GetScriptForDestination(ScriptHash(destination_script_1)), output1, input1); CreateCreditAndSpend(keystore, GetScriptForDestination(ScriptHash(destination_script_2)), output2, input2); ReplaceRedeemScript(input2.vin[0].scriptSig, destination_script_1); - CheckWithFlag(output1, input1, 0, true); + CheckWithFlag(output1, input1, SCRIPT_VERIFY_NONE, true); CheckWithFlag(output1, input1, SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output1, input1, SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output1, input1, STANDARD_SCRIPT_VERIFY_FLAGS, true); - CheckWithFlag(output1, input2, 0, true); + CheckWithFlag(output1, input2, SCRIPT_VERIFY_NONE, true); CheckWithFlag(output1, input2, SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output1, input2, SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH, false); CheckWithFlag(output1, input2, STANDARD_SCRIPT_VERIFY_FLAGS, false); @@ -662,11 +699,11 @@ BOOST_AUTO_TEST_CASE(test_witness) // Normal pay-to-uncompressed-pubkey. CreateCreditAndSpend(keystore, scriptPubkey1L, output1, input1); CreateCreditAndSpend(keystore, scriptPubkey2L, output2, input2); - CheckWithFlag(output1, input1, 0, true); + CheckWithFlag(output1, input1, SCRIPT_VERIFY_NONE, true); CheckWithFlag(output1, input1, SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output1, input1, SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output1, input1, STANDARD_SCRIPT_VERIFY_FLAGS, true); - CheckWithFlag(output1, input2, 0, false); + CheckWithFlag(output1, input2, SCRIPT_VERIFY_NONE, false); CheckWithFlag(output1, input2, SCRIPT_VERIFY_P2SH, false); CheckWithFlag(output1, input2, SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH, false); CheckWithFlag(output1, input2, STANDARD_SCRIPT_VERIFY_FLAGS, false); @@ -675,11 +712,11 @@ BOOST_AUTO_TEST_CASE(test_witness) CreateCreditAndSpend(keystore, GetScriptForDestination(ScriptHash(scriptPubkey1L)), output1, input1); CreateCreditAndSpend(keystore, GetScriptForDestination(ScriptHash(scriptPubkey2L)), output2, input2); ReplaceRedeemScript(input2.vin[0].scriptSig, scriptPubkey1L); - CheckWithFlag(output1, input1, 0, true); + CheckWithFlag(output1, input1, SCRIPT_VERIFY_NONE, true); CheckWithFlag(output1, input1, SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output1, input1, SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output1, input1, STANDARD_SCRIPT_VERIFY_FLAGS, true); - CheckWithFlag(output1, input2, 0, true); + CheckWithFlag(output1, input2, SCRIPT_VERIFY_NONE, true); CheckWithFlag(output1, input2, SCRIPT_VERIFY_P2SH, false); CheckWithFlag(output1, input2, SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH, false); CheckWithFlag(output1, input2, STANDARD_SCRIPT_VERIFY_FLAGS, false); @@ -694,19 +731,19 @@ BOOST_AUTO_TEST_CASE(test_witness) // Normal 2-of-2 multisig CreateCreditAndSpend(keystore, scriptMulti, output1, input1, false); - CheckWithFlag(output1, input1, 0, false); + CheckWithFlag(output1, input1, SCRIPT_VERIFY_NONE, false); CreateCreditAndSpend(keystore2, scriptMulti, output2, input2, false); - CheckWithFlag(output2, input2, 0, false); + CheckWithFlag(output2, input2, SCRIPT_VERIFY_NONE, false); BOOST_CHECK(*output1 == *output2); UpdateInput(input1.vin[0], CombineSignatures(input1, input2, output1)); CheckWithFlag(output1, input1, STANDARD_SCRIPT_VERIFY_FLAGS, true); // P2SH 2-of-2 multisig CreateCreditAndSpend(keystore, GetScriptForDestination(ScriptHash(scriptMulti)), output1, input1, false); - CheckWithFlag(output1, input1, 0, true); + CheckWithFlag(output1, input1, SCRIPT_VERIFY_NONE, true); CheckWithFlag(output1, input1, SCRIPT_VERIFY_P2SH, false); CreateCreditAndSpend(keystore2, GetScriptForDestination(ScriptHash(scriptMulti)), output2, input2, false); - CheckWithFlag(output2, input2, 0, true); + CheckWithFlag(output2, input2, SCRIPT_VERIFY_NONE, true); CheckWithFlag(output2, input2, SCRIPT_VERIFY_P2SH, false); BOOST_CHECK(*output1 == *output2); UpdateInput(input1.vin[0], CombineSignatures(input1, input2, output1)); @@ -715,10 +752,10 @@ BOOST_AUTO_TEST_CASE(test_witness) // Witness 2-of-2 multisig CreateCreditAndSpend(keystore, destination_script_multi, output1, input1, false); - CheckWithFlag(output1, input1, 0, true); + CheckWithFlag(output1, input1, SCRIPT_VERIFY_NONE, true); CheckWithFlag(output1, input1, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false); CreateCreditAndSpend(keystore2, destination_script_multi, output2, input2, false); - CheckWithFlag(output2, input2, 0, true); + CheckWithFlag(output2, input2, SCRIPT_VERIFY_NONE, true); CheckWithFlag(output2, input2, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false); BOOST_CHECK(*output1 == *output2); UpdateInput(input1.vin[0], CombineSignatures(input1, input2, output1)); @@ -779,21 +816,21 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) t.vout[0].nValue = nDustThreshold; CheckIsStandard(t); - // Disallowed nVersion - t.nVersion = -1; + // Disallowed version + t.version = std::numeric_limits<uint32_t>::max(); CheckIsNotStandard(t, "version"); - t.nVersion = 0; + t.version = 0; CheckIsNotStandard(t, "version"); - t.nVersion = TX_MAX_STANDARD_VERSION + 1; + t.version = TX_MAX_STANDARD_VERSION + 1; CheckIsNotStandard(t, "version"); - // Allowed nVersion - t.nVersion = 1; + // Allowed version + t.version = 1; CheckIsStandard(t); - t.nVersion = 2; + t.version = 2; CheckIsStandard(t); // Check dust with odd relay fee to verify rounding: @@ -904,15 +941,15 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) t.vin.clear(); t.vin.resize(2438); // size per input (empty scriptSig): 41 bytes t.vout[0].scriptPubKey = CScript() << OP_RETURN << std::vector<unsigned char>(19, 0); // output size: 30 bytes - // tx header: 12 bytes => 48 vbytes - // 2438 inputs: 2438*41 = 99958 bytes => 399832 vbytes - // 1 output: 30 bytes => 120 vbytes - // =============================== - // total: 400000 vbytes + // tx header: 12 bytes => 48 weight units + // 2438 inputs: 2438*41 = 99958 bytes => 399832 weight units + // 1 output: 30 bytes => 120 weight units + // ====================================== + // total: 400000 weight units BOOST_CHECK_EQUAL(GetTransactionWeight(CTransaction(t)), 400000); CheckIsStandard(t); - // increase output size by one byte, so we end up with 400004 vbytes + // increase output size by one byte, so we end up with 400004 weight units t.vout[0].scriptPubKey = CScript() << OP_RETURN << std::vector<unsigned char>(20, 0); // output size: 31 bytes BOOST_CHECK_EQUAL(GetTransactionWeight(CTransaction(t)), 400004); CheckIsNotStandard(t, "tx-size"); diff --git a/src/test/txpackage_tests.cpp b/src/test/txpackage_tests.cpp index 55e0c5f285..478121cc6f 100644 --- a/src/test/txpackage_tests.cpp +++ b/src/test/txpackage_tests.cpp @@ -6,6 +6,7 @@ #include <key_io.h> #include <policy/packages.h> #include <policy/policy.h> +#include <policy/rbf.h> #include <primitives/transaction.h> #include <script/script.h> #include <serialize.h> @@ -523,7 +524,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) CKey child_key = GenerateRandomKey(); CScript child_locking_script = GetScriptForDestination(WitnessV0KeyHash(child_key.GetPubKey())); CMutableTransaction mtx_child1; - mtx_child1.nVersion = 1; + mtx_child1.version = 1; mtx_child1.vin.resize(1); mtx_child1.vin[0].prevout.hash = ptx_parent->GetHash(); mtx_child1.vin[0].prevout.n = 0; @@ -651,7 +652,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) CTransactionRef ptx_grandparent2 = MakeTransactionRef(mtx_grandparent2); CMutableTransaction mtx_parent2_v1; - mtx_parent2_v1.nVersion = 1; + mtx_parent2_v1.version = 1; mtx_parent2_v1.vin.resize(1); mtx_parent2_v1.vin[0].prevout.hash = ptx_grandparent2->GetHash(); mtx_parent2_v1.vin[0].prevout.n = 0; @@ -938,4 +939,147 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); } } + +BOOST_FIXTURE_TEST_CASE(package_rbf_tests, TestChain100Setup) +{ + mineBlocks(5); + LOCK(::cs_main); + size_t expected_pool_size = m_node.mempool->size(); + CKey child_key{GenerateRandomKey()}; + CScript parent_spk = GetScriptForDestination(WitnessV0KeyHash(child_key.GetPubKey())); + CKey grandchild_key{GenerateRandomKey()}; + CScript child_spk = GetScriptForDestination(WitnessV0KeyHash(grandchild_key.GetPubKey())); + + const CAmount coinbase_value{50 * COIN}; + // Test that de-duplication works. This is not actually package rbf. + { + // 1 parent paying 200sat, 1 child paying 300sat + Package package1; + // 1 parent paying 200sat, 1 child paying 500sat + Package package2; + // Package1 and package2 have the same parent. The children conflict. + auto mtx_parent = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[0], /*input_vout=*/0, + /*input_height=*/0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/parent_spk, + /*output_amount=*/coinbase_value - low_fee_amt, /*submit=*/false); + CTransactionRef tx_parent = MakeTransactionRef(mtx_parent); + package1.push_back(tx_parent); + package2.push_back(tx_parent); + + CTransactionRef tx_child_1 = MakeTransactionRef(CreateValidMempoolTransaction(tx_parent, 0, 101, child_key, child_spk, coinbase_value - low_fee_amt - 300, false)); + package1.push_back(tx_child_1); + CTransactionRef tx_child_2 = MakeTransactionRef(CreateValidMempoolTransaction(tx_parent, 0, 101, child_key, child_spk, coinbase_value - low_fee_amt - 500, false)); + package2.push_back(tx_child_2); + + LOCK(m_node.mempool->cs); + const auto submit1 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package1, /*test_accept=*/false, std::nullopt); + if (auto err_1{CheckPackageMempoolAcceptResult(package1, submit1, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_1.value()); + } + + // Check precise ResultTypes and mempool size. We know it_parent_1 and it_child_1 exist from above call + auto it_parent_1 = submit1.m_tx_results.find(tx_parent->GetWitnessHash()); + auto it_child_1 = submit1.m_tx_results.find(tx_child_1->GetWitnessHash()); + BOOST_CHECK_EQUAL(it_parent_1->second.m_result_type, MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK_EQUAL(it_child_1->second.m_result_type, MempoolAcceptResult::ResultType::VALID); + expected_pool_size += 2; + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + + const auto submit2 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package2, /*test_accept=*/false, std::nullopt); + if (auto err_2{CheckPackageMempoolAcceptResult(package2, submit2, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_2.value()); + } + + // Check precise ResultTypes and mempool size. We know it_parent_2 and it_child_2 exist from above call + auto it_parent_2 = submit2.m_tx_results.find(tx_parent->GetWitnessHash()); + auto it_child_2 = submit2.m_tx_results.find(tx_child_2->GetWitnessHash()); + BOOST_CHECK_EQUAL(it_parent_2->second.m_result_type, MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + BOOST_CHECK_EQUAL(it_child_2->second.m_result_type, MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + + // child1 has been replaced + BOOST_CHECK(!m_node.mempool->exists(GenTxid::Txid(tx_child_1->GetHash()))); + } + + // Test package rbf. + { + CTransactionRef tx_parent_1 = MakeTransactionRef(CreateValidMempoolTransaction( + m_coinbase_txns[1], /*input_vout=*/0, /*input_height=*/0, + coinbaseKey, parent_spk, coinbase_value - 200, /*submit=*/false)); + CTransactionRef tx_child_1 = MakeTransactionRef(CreateValidMempoolTransaction( + tx_parent_1, /*input_vout=*/0, /*input_height=*/101, + child_key, child_spk, coinbase_value - 400, /*submit=*/false)); + + CTransactionRef tx_parent_2 = MakeTransactionRef(CreateValidMempoolTransaction( + m_coinbase_txns[1], /*input_vout=*/0, /*input_height=*/0, + coinbaseKey, parent_spk, coinbase_value - 800, /*submit=*/false)); + CTransactionRef tx_child_2 = MakeTransactionRef(CreateValidMempoolTransaction( + tx_parent_2, /*input_vout=*/0, /*input_height=*/101, + child_key, child_spk, coinbase_value - 800 - 200, /*submit=*/false)); + + CTransactionRef tx_parent_3 = MakeTransactionRef(CreateValidMempoolTransaction( + m_coinbase_txns[1], /*input_vout=*/0, /*input_height=*/0, + coinbaseKey, parent_spk, coinbase_value - 199, /*submit=*/false)); + CTransactionRef tx_child_3 = MakeTransactionRef(CreateValidMempoolTransaction( + tx_parent_3, /*input_vout=*/0, /*input_height=*/101, + child_key, child_spk, coinbase_value - 199 - 1300, /*submit=*/false)); + + // In all packages, the parents conflict with each other + BOOST_CHECK(tx_parent_1->GetHash() != tx_parent_2->GetHash() && tx_parent_2->GetHash() != tx_parent_3->GetHash()); + + // 1 parent paying 200sat, 1 child paying 200sat. + Package package1{tx_parent_1, tx_child_1}; + // 1 parent paying 800sat, 1 child paying 200sat. + Package package2{tx_parent_2, tx_child_2}; + // 1 parent paying 199sat, 1 child paying 1300sat. + Package package3{tx_parent_3, tx_child_3}; + + const auto submit1 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package1, false, std::nullopt); + if (auto err_1{CheckPackageMempoolAcceptResult(package1, submit1, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_1.value()); + } + auto it_parent_1 = submit1.m_tx_results.find(tx_parent_1->GetWitnessHash()); + auto it_child_1 = submit1.m_tx_results.find(tx_child_1->GetWitnessHash()); + BOOST_CHECK_EQUAL(it_parent_1->second.m_result_type, MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK_EQUAL(it_child_1->second.m_result_type, MempoolAcceptResult::ResultType::VALID); + expected_pool_size += 2; + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + + // This replacement is actually not package rbf; the parent carries enough fees + // to replace the entire package on its own. + const auto submit2 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package2, false, std::nullopt); + if (auto err_2{CheckPackageMempoolAcceptResult(package2, submit2, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_2.value()); + } + auto it_parent_2 = submit2.m_tx_results.find(tx_parent_2->GetWitnessHash()); + auto it_child_2 = submit2.m_tx_results.find(tx_child_2->GetWitnessHash()); + BOOST_CHECK_EQUAL(it_parent_2->second.m_result_type, MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK_EQUAL(it_child_2->second.m_result_type, MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + + // Package RBF, in which the replacement transaction's child sponsors the fees to meet RBF feerate rules + const auto submit3 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package3, false, std::nullopt); + if (auto err_3{CheckPackageMempoolAcceptResult(package3, submit3, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_3.value()); + } + auto it_parent_3 = submit3.m_tx_results.find(tx_parent_3->GetWitnessHash()); + auto it_child_3 = submit3.m_tx_results.find(tx_child_3->GetWitnessHash()); + BOOST_CHECK_EQUAL(it_parent_3->second.m_result_type, MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK_EQUAL(it_child_3->second.m_result_type, MempoolAcceptResult::ResultType::VALID); + + // package3 was considered as a package to replace both package2 transactions + BOOST_CHECK(it_parent_3->second.m_replaced_transactions.size() == 2); + BOOST_CHECK(it_child_3->second.m_replaced_transactions.empty()); + + std::vector<Wtxid> expected_package3_wtxids({tx_parent_3->GetWitnessHash(), tx_child_3->GetWitnessHash()}); + const auto package3_total_vsize{GetVirtualTransactionSize(*tx_parent_3) + GetVirtualTransactionSize(*tx_child_3)}; + BOOST_CHECK(it_parent_3->second.m_wtxids_fee_calculations.value() == expected_package3_wtxids); + BOOST_CHECK(it_child_3->second.m_wtxids_fee_calculations.value() == expected_package3_wtxids); + BOOST_CHECK_EQUAL(it_parent_3->second.m_effective_feerate.value().GetFee(package3_total_vsize), 199 + 1300); + BOOST_CHECK_EQUAL(it_child_3->second.m_effective_feerate.value().GetFee(package3_total_vsize), 199 + 1300); + + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + } + +} BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidation_tests.cpp b/src/test/txvalidation_tests.cpp index 95583b53bf..f429f94a2f 100644 --- a/src/test/txvalidation_tests.cpp +++ b/src/test/txvalidation_tests.cpp @@ -27,7 +27,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_reject_coinbase, TestChain100Setup) CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; CMutableTransaction coinbaseTx; - coinbaseTx.nVersion = 1; + coinbaseTx.version = 1; coinbaseTx.vin.resize(1); coinbaseTx.vout.resize(1); coinbaseTx.vin[0].scriptSig = CScript() << OP_11 << OP_EQUAL; @@ -72,11 +72,11 @@ static inline std::vector<CPubKey> random_keys(size_t num_keys) { return keys; } -// Creates a placeholder tx (not valid) with 25 outputs. Specify the nVersion and the inputs. +// Creates a placeholder tx (not valid) with 25 outputs. Specify the version and the inputs. static inline CTransactionRef make_tx(const std::vector<COutPoint>& inputs, int32_t version) { CMutableTransaction mtx = CMutableTransaction{}; - mtx.nVersion = version; + mtx.version = version; mtx.vin.resize(inputs.size()); mtx.vout.resize(25); for (size_t i{0}; i < inputs.size(); ++i) { @@ -286,7 +286,7 @@ BOOST_FIXTURE_TEST_CASE(version3_tests, RegTestingSetup) script_multisig << OP_2 << OP_CHECKMULTISIG; { CMutableTransaction mtx_many_sigops = CMutableTransaction{}; - mtx_many_sigops.nVersion = 3; + mtx_many_sigops.version = TRUC_VERSION; for (const auto& outpoint : multisig_outpoints) { mtx_many_sigops.vin.emplace_back(outpoint); mtx_many_sigops.vin.back().scriptWitness.stack.emplace_back(script_multisig.begin(), script_multisig.end()); diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 790eabc7c1..78ef96a15d 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -46,7 +46,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, Dersig100Setup) spends.resize(2); for (int i = 0; i < 2; i++) { - spends[i].nVersion = 1; + spends[i].version = 1; spends[i].vin.resize(1); spends[i].vin[0].prevout.hash = m_coinbase_txns[0]->GetHash(); spends[i].vin[0].prevout.n = 0; @@ -181,7 +181,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, Dersig100Setup) // coinbase tx. CMutableTransaction spend_tx; - spend_tx.nVersion = 1; + spend_tx.version = 1; spend_tx.vin.resize(1); spend_tx.vin[0].prevout.hash = m_coinbase_txns[0]->GetHash(); spend_tx.vin[0].prevout.n = 0; @@ -243,7 +243,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, Dersig100Setup) // then test validity with P2SH. { CMutableTransaction invalid_under_p2sh_tx; - invalid_under_p2sh_tx.nVersion = 1; + invalid_under_p2sh_tx.version = 1; invalid_under_p2sh_tx.vin.resize(1); invalid_under_p2sh_tx.vin[0].prevout.hash = spend_tx.GetHash(); invalid_under_p2sh_tx.vin[0].prevout.n = 0; @@ -259,7 +259,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, Dersig100Setup) // Test CHECKLOCKTIMEVERIFY { CMutableTransaction invalid_with_cltv_tx; - invalid_with_cltv_tx.nVersion = 1; + invalid_with_cltv_tx.version = 1; invalid_with_cltv_tx.nLockTime = 100; invalid_with_cltv_tx.vin.resize(1); invalid_with_cltv_tx.vin[0].prevout.hash = spend_tx.GetHash(); @@ -288,7 +288,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, Dersig100Setup) // TEST CHECKSEQUENCEVERIFY { CMutableTransaction invalid_with_csv_tx; - invalid_with_csv_tx.nVersion = 2; + invalid_with_csv_tx.version = 2; invalid_with_csv_tx.vin.resize(1); invalid_with_csv_tx.vin[0].prevout.hash = spend_tx.GetHash(); invalid_with_csv_tx.vin[0].prevout.n = 3; @@ -319,7 +319,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, Dersig100Setup) // for the same tx with a different witness. { CMutableTransaction valid_with_witness_tx; - valid_with_witness_tx.nVersion = 1; + valid_with_witness_tx.version = 1; valid_with_witness_tx.vin.resize(1); valid_with_witness_tx.vin[0].prevout.hash = spend_tx.GetHash(); valid_with_witness_tx.vin[0].prevout.n = 1; @@ -344,7 +344,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, Dersig100Setup) // Test a transaction with multiple inputs. CMutableTransaction tx; - tx.nVersion = 1; + tx.version = 1; tx.vin.resize(2); tx.vin[0].prevout.hash = spend_tx.GetHash(); tx.vin[0].prevout.n = 0; diff --git a/src/test/uint256_tests.cpp b/src/test/uint256_tests.cpp index 5746961550..b7892a2139 100644 --- a/src/test/uint256_tests.cpp +++ b/src/test/uint256_tests.cpp @@ -267,6 +267,22 @@ BOOST_AUTO_TEST_CASE( conversion ) BOOST_AUTO_TEST_CASE( operator_with_self ) { + +/* Clang 16 and earlier detects v -= v and v /= v as self-assignments + to 0 and 1 respectively. + See: https://github.com/llvm/llvm-project/issues/42469 + and the fix in commit c5302325b2a62d77cf13dd16cd5c19141862fed0 . + + This makes some sense for arithmetic classes, but could be considered a bug + elsewhere. Disable the warning here so that the code can be tested, but the + warning should remain on as there will likely always be a better way to + express this. +*/ + +#if defined(__clang__) +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wself-assign-overloaded" +#endif arith_uint256 v = UintToArith256(uint256S("02")); v *= v; BOOST_CHECK(v == UintToArith256(uint256S("04"))); @@ -276,6 +292,9 @@ BOOST_AUTO_TEST_CASE( operator_with_self ) BOOST_CHECK(v == UintToArith256(uint256S("02"))); v -= v; BOOST_CHECK(v == UintToArith256(uint256S("0"))); +#if defined(__clang__) +# pragma clang diagnostic pop +#endif } BOOST_AUTO_TEST_CASE(parse) diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h index ff95e64b7e..03b44fc894 100644 --- a/src/test/util/chainstate.h +++ b/src/test/util/chainstate.h @@ -56,7 +56,7 @@ CreateAndActivateUTXOSnapshot( // FILE* infile{fsbridge::fopen(snapshot_path, "rb")}; AutoFile auto_infile{infile}; - node::SnapshotMetadata metadata; + node::SnapshotMetadata metadata{node.chainman->GetParams().MessageStart()}; auto_infile >> metadata; malleation(auto_infile, metadata); diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index fd07931716..cc7b2d6546 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -31,6 +31,7 @@ #include <node/miner.h> #include <node/peerman_args.h> #include <node/validation_cache_args.h> +#include <node/warnings.h> #include <noui.h> #include <policy/fees.h> #include <policy/fees_args.h> @@ -182,6 +183,7 @@ BasicTestingSetup::BasicTestingSetup(const ChainType chainType, const std::vecto InitLogging(*m_node.args); AppInitParameterInteraction(*m_node.args); LogInstance().StartLogging(); + m_node.warnings = std::make_unique<node::Warnings>(); m_node.kernel = std::make_unique<kernel::Context>(); m_node.ecc_context = std::make_unique<ECC_Context>(); SetupEnvironment(); @@ -227,16 +229,19 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, const std::vecto m_node.validation_signals = std::make_unique<ValidationSignals>(std::make_unique<SerialTaskRunner>(*m_node.scheduler)); m_node.fee_estimator = std::make_unique<CBlockPolicyEstimator>(FeeestPath(*m_node.args), DEFAULT_ACCEPT_STALE_FEE_ESTIMATES); - m_node.mempool = std::make_unique<CTxMemPool>(MemPoolOptionsForTest(m_node)); + bilingual_str error{}; + m_node.mempool = std::make_unique<CTxMemPool>(MemPoolOptionsForTest(m_node), error); + Assert(error.empty()); + m_node.warnings = std::make_unique<node::Warnings>(); m_cache_sizes = CalculateCacheSizes(m_args); - m_node.notifications = std::make_unique<KernelNotifications>(*Assert(m_node.shutdown), m_node.exit_status); + m_node.notifications = std::make_unique<KernelNotifications>(*Assert(m_node.shutdown), m_node.exit_status, *Assert(m_node.warnings)); const ChainstateManager::Options chainman_opts{ .chainparams = chainparams, .datadir = m_args.GetDataDirNet(), - .check_block_index = true, + .check_block_index = 1, .notifications = *m_node.notifications, .signals = m_node.validation_signals.get(), .worker_threads_num = 2, @@ -276,8 +281,8 @@ void ChainTestingSetup::LoadVerifyActivateChainstate() options.mempool = Assert(m_node.mempool.get()); options.block_tree_db_in_memory = m_block_tree_db_in_memory; options.coins_db_in_memory = m_coins_db_in_memory; - options.reindex = chainman.m_blockman.m_reindexing; - options.reindex_chainstate = m_args.GetBoolArg("-reindex-chainstate", false); + options.wipe_block_tree_db = m_args.GetBoolArg("-reindex", false); + options.wipe_chainstate_db = m_args.GetBoolArg("-reindex", false) || m_args.GetBoolArg("-reindex-chainstate", false); options.prune = chainman.m_blockman.IsPruneMode(); options.check_blocks = m_args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); options.check_level = m_args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL); @@ -320,7 +325,8 @@ TestingSetup::TestingSetup( peerman_opts.deterministic_rng = true; m_node.peerman = PeerManager::make(*m_node.connman, *m_node.addrman, m_node.banman.get(), *m_node.chainman, - *m_node.mempool, peerman_opts); + *m_node.mempool, *m_node.warnings, + peerman_opts); { CConnman::Options options; @@ -368,7 +374,8 @@ CBlock TestChain100Setup::CreateBlock( const CScript& scriptPubKey, Chainstate& chainstate) { - CBlock block = BlockAssembler{chainstate, nullptr}.CreateNewBlock(scriptPubKey)->block; + BlockAssembler::Options options; + CBlock block = BlockAssembler{chainstate, nullptr, options}.CreateNewBlock(scriptPubKey)->block; Assert(block.vtx.size() == 1); for (const CMutableTransaction& tx : txns) { diff --git a/src/test/util/transaction_utils.cpp b/src/test/util/transaction_utils.cpp index 7e5bb30a2c..300caa577c 100644 --- a/src/test/util/transaction_utils.cpp +++ b/src/test/util/transaction_utils.cpp @@ -9,7 +9,7 @@ CMutableTransaction BuildCreditingTransaction(const CScript& scriptPubKey, int nValue) { CMutableTransaction txCredit; - txCredit.nVersion = 1; + txCredit.version = 1; txCredit.nLockTime = 0; txCredit.vin.resize(1); txCredit.vout.resize(1); @@ -25,7 +25,7 @@ CMutableTransaction BuildCreditingTransaction(const CScript& scriptPubKey, int n CMutableTransaction BuildSpendingTransaction(const CScript& scriptSig, const CScriptWitness& scriptWitness, const CTransaction& txCredit) { CMutableTransaction txSpend; - txSpend.nVersion = 1; + txSpend.version = 1; txSpend.nLockTime = 0; txSpend.vin.resize(1); txSpend.vout.resize(1); diff --git a/src/test/util/txmempool.cpp b/src/test/util/txmempool.cpp index 870cdd0b32..94d50bba50 100644 --- a/src/test/util/txmempool.cpp +++ b/src/test/util/txmempool.cpp @@ -7,6 +7,7 @@ #include <chainparams.h> #include <node/context.h> #include <node/mempool_args.h> +#include <policy/rbf.h> #include <policy/v3_policy.h> #include <txmempool.h> #include <util/check.h> @@ -68,6 +69,28 @@ std::optional<std::string> CheckPackageMempoolAcceptResult(const Package& txns, return strprintf("tx %s unexpectedly failed: %s", wtxid.ToString(), atmp_result.m_state.ToString()); } + // Each subpackage is allowed MAX_REPLACEMENT_CANDIDATES replacements (only checking individually here) + if (atmp_result.m_replaced_transactions.size() > MAX_REPLACEMENT_CANDIDATES) { + return strprintf("tx %s result replaced too many transactions", + wtxid.ToString()); + } + + // Replacements can't happen for subpackages larger than 2 + if (!atmp_result.m_replaced_transactions.empty() && + atmp_result.m_wtxids_fee_calculations.has_value() && atmp_result.m_wtxids_fee_calculations.value().size() > 2) { + return strprintf("tx %s was part of a too-large package RBF subpackage", + wtxid.ToString()); + } + + if (!atmp_result.m_replaced_transactions.empty() && mempool) { + LOCK(mempool->cs); + // If replacements occurred and it used 2 transactions, this is a package RBF and should result in a cluster of size 2 + if (atmp_result.m_wtxids_fee_calculations.has_value() && atmp_result.m_wtxids_fee_calculations.value().size() == 2) { + const auto cluster = mempool->GatherClusters({tx->GetHash()}); + if (cluster.size() != 2) return strprintf("tx %s has too many ancestors or descendants for a package rbf", wtxid.ToString()); + } + } + // m_vsize and m_base_fees should exist iff the result was VALID or MEMPOOL_ENTRY const bool mempool_entry{atmp_result.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY}; if (atmp_result.m_base_fees.has_value() != (valid || mempool_entry)) { @@ -108,6 +131,11 @@ std::optional<std::string> CheckPackageMempoolAcceptResult(const Package& txns, return strprintf("wtxid %s should not be in mempool", wtxid.ToString()); } } + for (const auto& tx_ref : atmp_result.m_replaced_transactions) { + if (mempool->exists(GenTxid::Txid(tx_ref->GetHash()))) { + return strprintf("tx %s should not be in mempool as it was replaced", tx_ref->GetWitnessHash().ToString()); + } + } } } return std::nullopt; @@ -118,21 +146,27 @@ void CheckMempoolV3Invariants(const CTxMemPool& tx_pool) LOCK(tx_pool.cs); for (const auto& tx_info : tx_pool.infoAll()) { const auto& entry = *Assert(tx_pool.GetEntry(tx_info.tx->GetHash())); - if (tx_info.tx->nVersion == 3) { + if (tx_info.tx->version == TRUC_VERSION) { + // Check that special maximum virtual size is respected + Assert(entry.GetTxSize() <= V3_MAX_VSIZE); + // Check that special v3 ancestor/descendant limits and rules are always respected Assert(entry.GetCountWithDescendants() <= V3_DESCENDANT_LIMIT); Assert(entry.GetCountWithAncestors() <= V3_ANCESTOR_LIMIT); + Assert(entry.GetSizeWithDescendants() <= V3_MAX_VSIZE + V3_CHILD_MAX_VSIZE); + Assert(entry.GetSizeWithAncestors() <= V3_MAX_VSIZE + V3_CHILD_MAX_VSIZE); + // If this transaction has at least 1 ancestor, it's a "child" and has restricted weight. if (entry.GetCountWithAncestors() > 1) { Assert(entry.GetTxSize() <= V3_CHILD_MAX_VSIZE); // All v3 transactions must only have v3 unconfirmed parents. const auto& parents = entry.GetMemPoolParentsConst(); - Assert(parents.begin()->get().GetSharedTx()->nVersion == 3); + Assert(parents.begin()->get().GetSharedTx()->version == TRUC_VERSION); } } else if (entry.GetCountWithAncestors() > 1) { // All non-v3 transactions must only have non-v3 unconfirmed parents. for (const auto& parent : entry.GetMemPoolParentsConst()) { - Assert(parent.get().GetSharedTx()->nVersion != 3); + Assert(parent.get().GetSharedTx()->version != TRUC_VERSION); } } } diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 9a2add748e..a371753adf 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -3,8 +3,10 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <clientversion.h> +#include <common/signmessage.h> // For MessageSign(), MessageVerify(), MESSAGE_MAGIC #include <hash.h> // For Hash() #include <key.h> // For CKey +#include <script/parsing.h> #include <sync.h> #include <test/util/random.h> #include <test/util/setup_common.h> @@ -12,11 +14,9 @@ #include <util/bitdeque.h> #include <util/fs.h> #include <util/fs_helpers.h> -#include <util/message.h> // For MessageSign(), MessageVerify(), MESSAGE_MAGIC #include <util/moneystr.h> #include <util/overflow.h> #include <util/readwritefile.h> -#include <util/spanparsing.h> #include <util/strencodings.h> #include <util/string.h> #include <util/time.h> @@ -45,6 +45,15 @@ #include <boost/test/unit_test.hpp> using namespace std::literals; +using util::Join; +using util::RemovePrefix; +using util::RemovePrefixView; +using util::ReplaceAll; +using util::Split; +using util::SplitString; +using util::TrimString; +using util::TrimStringView; + static const std::string STRING_WITH_EMBEDDED_NULL_CHAR{"1"s "\0" "1"s}; /* defined in logging.cpp */ @@ -1292,9 +1301,9 @@ static std::string SpanToStr(const Span<const char>& span) return std::string(span.begin(), span.end()); } -BOOST_AUTO_TEST_CASE(test_spanparsing) +BOOST_AUTO_TEST_CASE(test_script_parsing) { - using namespace spanparsing; + using namespace script; std::string input; Span<const char> sp; bool success; diff --git a/src/test/util_threadnames_tests.cpp b/src/test/util_threadnames_tests.cpp index 174052d5fa..efa0b2736b 100644 --- a/src/test/util_threadnames_tests.cpp +++ b/src/test/util_threadnames_tests.cpp @@ -13,6 +13,8 @@ #include <boost/test/unit_test.hpp> +using util::ToString; + BOOST_AUTO_TEST_SUITE(util_threadnames_tests) const std::string TEST_THREAD_NAME_BASE = "test_thread."; diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index 69f4e305ab..588ac60498 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -65,7 +65,8 @@ std::shared_ptr<CBlock> MinerTestingSetup::Block(const uint256& prev_hash) static int i = 0; static uint64_t time = Params().GenesisBlock().nTime; - auto ptemplate = BlockAssembler{m_node.chainman->ActiveChainstate(), m_node.mempool.get()}.CreateNewBlock(CScript{} << i++ << OP_TRUE); + BlockAssembler::Options options; + auto ptemplate = BlockAssembler{m_node.chainman->ActiveChainstate(), m_node.mempool.get(), options}.CreateNewBlock(CScript{} << i++ << OP_TRUE); auto pblock = std::make_shared<CBlock>(ptemplate->block); pblock->hashPrevBlock = prev_hash; pblock->nTime = ++time; @@ -329,7 +330,8 @@ BOOST_AUTO_TEST_CASE(witness_commitment_index) LOCK(Assert(m_node.chainman)->GetMutex()); CScript pubKey; pubKey << 1 << OP_TRUE; - auto ptemplate = BlockAssembler{m_node.chainman->ActiveChainstate(), m_node.mempool.get()}.CreateNewBlock(pubKey); + BlockAssembler::Options options; + auto ptemplate = BlockAssembler{m_node.chainman->ActiveChainstate(), m_node.mempool.get(), options}.CreateNewBlock(pubKey); CBlock pblock = ptemplate->block; CTxOut witness; diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 4bf66a55eb..1641c4cd22 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -226,10 +226,13 @@ struct SnapshotTestSetup : TestChain100Setup { // A UTXO is missing but count is correct metadata.m_coins_count -= 1; - COutPoint outpoint; + Txid txid; + auto_infile >> txid; + // coins size + (void)ReadCompactSize(auto_infile); + // vout index + (void)ReadCompactSize(auto_infile); Coin coin; - - auto_infile >> outpoint; auto_infile >> coin; })); @@ -375,7 +378,7 @@ struct SnapshotTestSetup : TestChain100Setup { LOCK(::cs_main); chainman.ResetChainstates(); BOOST_CHECK_EQUAL(chainman.GetAll().size(), 0); - m_node.notifications = std::make_unique<KernelNotifications>(*Assert(m_node.shutdown), m_node.exit_status); + m_node.notifications = std::make_unique<KernelNotifications>(*Assert(m_node.shutdown), m_node.exit_status, *Assert(m_node.warnings)); const ChainstateManager::Options chainman_opts{ .chainparams = ::Params(), .datadir = chainman.m_options.datadir, diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 442c1c4d42..4f79644c8d 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -42,6 +42,10 @@ #include <event2/thread.h> #include <event2/util.h> +using util::ReplaceAll; +using util::SplitString; +using util::ToString; + /** Default control ip and port */ const std::string DEFAULT_TOR_CONTROL = "127.0.0.1:" + ToString(DEFAULT_TOR_CONTROL_PORT); /** Tor cookie size (from control-spec.txt) */ diff --git a/src/txmempool.cpp b/src/txmempool.cpp index c845944d32..10674c07ac 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -16,6 +16,7 @@ #include <policy/settings.h> #include <random.h> #include <reverse_iterator.h> +#include <tinyformat.h> #include <util/check.h> #include <util/feefrac.h> #include <util/moneystr.h> @@ -26,6 +27,7 @@ #include <util/translation.h> #include <validationinterface.h> +#include <algorithm> #include <cmath> #include <numeric> #include <optional> @@ -395,8 +397,19 @@ void CTxMemPoolEntry::UpdateAncestorState(int32_t modifySize, CAmount modifyFee, assert(int(nSigOpCostWithAncestors) >= 0); } -CTxMemPool::CTxMemPool(const Options& opts) - : m_opts{opts} +//! Clamp option values and populate the error if options are not valid. +static CTxMemPool::Options&& Flatten(CTxMemPool::Options&& opts, bilingual_str& error) +{ + opts.check_ratio = std::clamp<int>(opts.check_ratio, 0, 1'000'000); + int64_t descendant_limit_bytes = opts.limits.descendant_size_vbytes * 40; + if (opts.max_size_bytes < 0 || opts.max_size_bytes < descendant_limit_bytes) { + error = strprintf(_("-maxmempool must be at least %d MB"), std::ceil(descendant_limit_bytes / 1'000'000.0)); + } + return std::move(opts); +} + +CTxMemPool::CTxMemPool(Options opts, bilingual_str& error) + : m_opts{Flatten(std::move(opts), error)} { } diff --git a/src/txmempool.h b/src/txmempool.h index c9f6cdbfac..52f186f0ff 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -43,6 +43,8 @@ class CChain; class ValidationSignals; +struct bilingual_str; + /** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */ static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; @@ -442,7 +444,7 @@ public: * accepting transactions becomes O(N^2) where N is the number of transactions * in the pool. */ - explicit CTxMemPool(const Options& opts); + explicit CTxMemPool(Options opts, bilingual_str& error); /** * If sanity-checking is turned on, check makes sure the pool is diff --git a/src/txorphanage.cpp b/src/txorphanage.cpp index 13ef96f3be..3eaf53939d 100644 --- a/src/txorphanage.cpp +++ b/src/txorphanage.cpp @@ -8,13 +8,14 @@ #include <logging.h> #include <policy/policy.h> #include <primitives/transaction.h> +#include <util/time.h> #include <cassert> -/** Expiration time for orphan transactions in seconds */ -static constexpr int64_t ORPHAN_TX_EXPIRE_TIME = 20 * 60; -/** Minimum time between orphan transactions expire time checks in seconds */ -static constexpr int64_t ORPHAN_TX_EXPIRE_INTERVAL = 5 * 60; +/** Expiration time for orphan transactions */ +static constexpr auto ORPHAN_TX_EXPIRE_TIME{20min}; +/** Minimum time between orphan transactions expire time checks */ +static constexpr auto ORPHAN_TX_EXPIRE_INTERVAL{5min}; bool TxOrphanage::AddTx(const CTransactionRef& tx, NodeId peer) @@ -40,7 +41,7 @@ bool TxOrphanage::AddTx(const CTransactionRef& tx, NodeId peer) return false; } - auto ret = m_orphans.emplace(wtxid, OrphanTx{tx, peer, GetTime() + ORPHAN_TX_EXPIRE_TIME, m_orphan_list.size()}); + auto ret = m_orphans.emplace(wtxid, OrphanTx{tx, peer, Now<NodeSeconds>() + ORPHAN_TX_EXPIRE_TIME, m_orphan_list.size()}); assert(ret.second); m_orphan_list.push_back(ret.first); for (const CTxIn& txin : tx->vin) { @@ -87,7 +88,7 @@ int TxOrphanage::EraseTxNoLock(const Wtxid& wtxid) // Time spent in orphanage = difference between current and entry time. // Entry time is equal to ORPHAN_TX_EXPIRE_TIME earlier than entry's expiry. LogPrint(BCLog::TXPACKAGES, " removed orphan tx %s (wtxid=%s) after %ds\n", txid.ToString(), wtxid.ToString(), - GetTime() + ORPHAN_TX_EXPIRE_TIME - it->second.nTimeExpire); + Ticks<std::chrono::seconds>(NodeClock::now() + ORPHAN_TX_EXPIRE_TIME - it->second.nTimeExpire)); m_orphan_list.pop_back(); m_orphans.erase(it); @@ -118,12 +119,11 @@ void TxOrphanage::LimitOrphans(unsigned int max_orphans, FastRandomContext& rng) LOCK(m_mutex); unsigned int nEvicted = 0; - static int64_t nNextSweep; - int64_t nNow = GetTime(); - if (nNextSweep <= nNow) { + auto nNow{Now<NodeSeconds>()}; + if (m_next_sweep <= nNow) { // Sweep out expired orphan pool entries: int nErased = 0; - int64_t nMinExpTime = nNow + ORPHAN_TX_EXPIRE_TIME - ORPHAN_TX_EXPIRE_INTERVAL; + auto nMinExpTime{nNow + ORPHAN_TX_EXPIRE_TIME - ORPHAN_TX_EXPIRE_INTERVAL}; std::map<Wtxid, OrphanTx>::iterator iter = m_orphans.begin(); while (iter != m_orphans.end()) { @@ -135,7 +135,7 @@ void TxOrphanage::LimitOrphans(unsigned int max_orphans, FastRandomContext& rng) } } // Sweep again 5 minutes after the next entry that expires in order to batch the linear scan. - nNextSweep = nMinExpTime + ORPHAN_TX_EXPIRE_INTERVAL; + m_next_sweep = nMinExpTime + ORPHAN_TX_EXPIRE_INTERVAL; if (nErased > 0) LogPrint(BCLog::TXPACKAGES, "Erased %d orphan tx due to expiration\n", nErased); } while (m_orphans.size() > max_orphans) diff --git a/src/txorphanage.h b/src/txorphanage.h index 5f9888c969..3054396b2d 100644 --- a/src/txorphanage.h +++ b/src/txorphanage.h @@ -9,6 +9,7 @@ #include <primitives/block.h> #include <primitives/transaction.h> #include <sync.h> +#include <util/time.h> #include <map> #include <set> @@ -73,7 +74,7 @@ protected: struct OrphanTx { CTransactionRef tx; NodeId fromPeer; - int64_t nTimeExpire; + NodeSeconds nTimeExpire; size_t list_pos; }; @@ -104,6 +105,9 @@ protected: /** Erase an orphan by wtxid */ int EraseTxNoLock(const Wtxid& wtxid) EXCLUSIVE_LOCKS_REQUIRED(m_mutex); + + /** Timestamp for the next scheduled sweep of expired orphans */ + NodeSeconds m_next_sweep GUARDED_BY(m_mutex){0s}; }; #endif // BITCOIN_TXORPHANAGE_H diff --git a/src/util/bitset.h b/src/util/bitset.h new file mode 100644 index 0000000000..6f9e808c37 --- /dev/null +++ b/src/util/bitset.h @@ -0,0 +1,527 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_BITSET_H +#define BITCOIN_UTIL_BITSET_H + +#include <util/check.h> + +#include <array> +#include <bit> +#include <cstdint> +#include <limits> +#include <type_traits> + +/* This file provides data types similar to std::bitset, but adds the following functionality: + * + * - Efficient iteration over all set bits (compatible with range-based for loops). + * - Efficient search for the first and last set bit (First() and Last()). + * - Efficient set subtraction: (a - b) implements "a and not b". + * - Efficient non-strict subset/superset testing: IsSubsetOf() and IsSupersetOf(). + * - Efficient set overlap testing: a.Overlaps(b) + * - Efficient construction of set containing 0..N-1 (S::Fill). + * - Efficient construction of a single set (S::Singleton). + * - Construction from initializer lists. + * + * Other differences: + * - BitSet<N> is a bitset that supports at least N elements, but may support more (Size() reports + * the actual number). Because the actual number is unpredictable, there are no operations that + * affect all positions (like std::bitset's operator~, flip(), or all()). + * - Various other unimplemented features. + */ + +namespace bitset_detail { + +/** Count the number of bits set in an unsigned integer type. */ +template<typename I> +unsigned inline constexpr PopCount(I v) +{ + static_assert(std::is_integral_v<I> && std::is_unsigned_v<I> && std::numeric_limits<I>::radix == 2); + constexpr auto BITS = std::numeric_limits<I>::digits; + // Algorithms from https://en.wikipedia.org/wiki/Hamming_weight#Efficient_implementation. + // These seem to be faster than std::popcount when compiling for non-SSE4 on x86_64. + if constexpr (BITS <= 32) { + v -= (v >> 1) & 0x55555555; + v = (v & 0x33333333) + ((v >> 2) & 0x33333333); + v = (v + (v >> 4)) & 0x0f0f0f0f; + if constexpr (BITS > 8) v += v >> 8; + if constexpr (BITS > 16) v += v >> 16; + return v & 0x3f; + } else { + static_assert(BITS <= 64); + v -= (v >> 1) & 0x5555555555555555; + v = (v & 0x3333333333333333) + ((v >> 2) & 0x3333333333333333); + v = (v + (v >> 4)) & 0x0f0f0f0f0f0f0f0f; + return (v * uint64_t{0x0101010101010101}) >> 56; + } +} + +/** A bitset implementation backed by a single integer of type I. */ +template<typename I> +class IntBitSet +{ + // Only binary, unsigned, integer, types allowed. + static_assert(std::is_integral_v<I> && std::is_unsigned_v<I> && std::numeric_limits<I>::radix == 2); + /** The maximum number of bits this bitset supports. */ + static constexpr unsigned MAX_SIZE = std::numeric_limits<I>::digits; + /** Integer whose bits represent this bitset. */ + I m_val; + /** Internal constructor with a given integer as contents. */ + IntBitSet(I val) noexcept : m_val{val} {} + /** Dummy type to return using end(). Only used for comparing with Iterator. */ + class IteratorEnd + { + friend class IntBitSet; + constexpr IteratorEnd() = default; + public: + constexpr IteratorEnd(const IteratorEnd&) = default; + }; + /** Iterator type returned by begin(), which efficiently iterates all 1 positions. */ + class Iterator + { + friend class IntBitSet; + I m_val; /**< The original integer's remaining bits. */ + unsigned m_pos; /** Last reported 1 position (if m_pos != 0). */ + constexpr Iterator(I val) noexcept : m_val(val), m_pos(0) + { + if (m_val != 0) m_pos = std::countr_zero(m_val); + } + public: + /** Do not allow external code to construct an Iterator. */ + Iterator() = delete; + // Copying is allowed. + constexpr Iterator(const Iterator&) noexcept = default; + constexpr Iterator& operator=(const Iterator&) noexcept = default; + /** Test whether we are done (can only compare with IteratorEnd). */ + constexpr friend bool operator==(const Iterator& a, const IteratorEnd&) noexcept + { + return a.m_val == 0; + } + /** Progress to the next 1 bit (only if != IteratorEnd). */ + constexpr Iterator& operator++() noexcept + { + Assume(m_val != 0); + m_val &= m_val - I{1U}; + if (m_val != 0) m_pos = std::countr_zero(m_val); + return *this; + } + /** Get the current bit position (only if != IteratorEnd). */ + constexpr unsigned operator*() const noexcept + { + Assume(m_val != 0); + return m_pos; + } + }; + +public: + /** Construct an all-zero bitset. */ + constexpr IntBitSet() noexcept : m_val{0} {} + /** Copy construct a bitset. */ + constexpr IntBitSet(const IntBitSet&) noexcept = default; + /** Construct from a list of values. */ + constexpr IntBitSet(std::initializer_list<unsigned> ilist) noexcept : m_val(0) + { + for (auto pos : ilist) Set(pos); + } + /** Copy assign a bitset. */ + constexpr IntBitSet& operator=(const IntBitSet&) noexcept = default; + /** Assign from a list of positions (which will be made true, all others false). */ + constexpr IntBitSet& operator=(std::initializer_list<unsigned> ilist) noexcept + { + m_val = 0; + for (auto pos : ilist) Set(pos); + return *this; + } + /** Construct a bitset with the singleton i. */ + static constexpr IntBitSet Singleton(unsigned i) noexcept + { + Assume(i < MAX_SIZE); + return IntBitSet(I(1U) << i); + } + /** Construct a bitset with bits 0..count-1 (inclusive) set to 1. */ + static constexpr IntBitSet Fill(unsigned count) noexcept + { + IntBitSet ret; + Assume(count <= MAX_SIZE); + if (count) ret.m_val = I(~I{0}) >> (MAX_SIZE - count); + return ret; + } + /** Set a bit to 1. */ + constexpr void Set(unsigned pos) noexcept + { + Assume(pos < MAX_SIZE); + m_val |= I{1U} << pos; + } + /** Set a bit to the specified value. */ + constexpr void Set(unsigned pos, bool val) noexcept + { + Assume(pos < MAX_SIZE); + m_val = (m_val & ~I(I{1U} << pos)) | (I(val) << pos); + } + /** Set a bit to 0. */ + constexpr void Reset(unsigned pos) noexcept + { + Assume(pos < MAX_SIZE); + m_val &= ~I(I{1U} << pos); + } + /** Retrieve a bit at the given position. */ + constexpr bool operator[](unsigned pos) const noexcept + { + Assume(pos < MAX_SIZE); + return (m_val >> pos) & 1U; + } + /** Compute the number of 1 bits in the bitset. */ + constexpr unsigned Count() const noexcept { return PopCount(m_val); } + /** Return the number of bits that this object holds. */ + static constexpr unsigned Size() noexcept { return MAX_SIZE; } + /** Check if all bits are 0. */ + constexpr bool None() const noexcept { return m_val == 0; } + /** Check if any bits are 1. */ + constexpr bool Any() const noexcept { return !None(); } + /** Return an object that iterates over all 1 bits (++ and * only allowed when != end()). */ + constexpr Iterator begin() const noexcept { return Iterator(m_val); } + /** Return a dummy object to compare Iterators with. */ + constexpr IteratorEnd end() const noexcept { return IteratorEnd(); } + /** Find the first element (requires Any()). */ + constexpr unsigned First() const noexcept + { + Assume(m_val != 0); + return std::countr_zero(m_val); + } + /** Find the last element (requires Any()). */ + constexpr unsigned Last() const noexcept + { + Assume(m_val != 0); + return std::bit_width(m_val) - 1; + } + /** Set this object's bits to be the binary AND between respective bits from this and a. */ + constexpr IntBitSet& operator|=(const IntBitSet& a) noexcept { m_val |= a.m_val; return *this; } + /** Set this object's bits to be the binary OR between respective bits from this and a. */ + constexpr IntBitSet& operator&=(const IntBitSet& a) noexcept { m_val &= a.m_val; return *this; } + /** Set this object's bits to be the binary AND NOT between respective bits from this and a. */ + constexpr IntBitSet& operator-=(const IntBitSet& a) noexcept { m_val &= ~a.m_val; return *this; } + /** Set this object's bits to be the binary XOR between respective bits from this as a. */ + constexpr IntBitSet& operator^=(const IntBitSet& a) noexcept { m_val ^= a.m_val; return *this; } + /** Check if the intersection between two sets is non-empty. */ + constexpr bool Overlaps(const IntBitSet& a) const noexcept { return m_val & a.m_val; } + /** Return an object with the binary AND between respective bits from a and b. */ + friend constexpr IntBitSet operator&(const IntBitSet& a, const IntBitSet& b) noexcept { return I(a.m_val & b.m_val); } + /** Return an object with the binary OR between respective bits from a and b. */ + friend constexpr IntBitSet operator|(const IntBitSet& a, const IntBitSet& b) noexcept { return I(a.m_val | b.m_val); } + /** Return an object with the binary AND NOT between respective bits from a and b. */ + friend constexpr IntBitSet operator-(const IntBitSet& a, const IntBitSet& b) noexcept { return I(a.m_val & ~b.m_val); } + /** Return an object with the binary XOR between respective bits from a and b. */ + friend constexpr IntBitSet operator^(const IntBitSet& a, const IntBitSet& b) noexcept { return I(a.m_val ^ b.m_val); } + /** Check if bitset a and bitset b are identical. */ + friend constexpr bool operator==(const IntBitSet& a, const IntBitSet& b) noexcept = default; + /** Check if bitset a is a superset of bitset b (= every 1 bit in b is also in a). */ + constexpr bool IsSupersetOf(const IntBitSet& a) const noexcept { return (a.m_val & ~m_val) == 0; } + /** Check if bitset a is a subset of bitset b (= every 1 bit in a is also in b). */ + constexpr bool IsSubsetOf(const IntBitSet& a) const noexcept { return (m_val & ~a.m_val) == 0; } + /** Swap two bitsets. */ + friend constexpr void swap(IntBitSet& a, IntBitSet& b) noexcept { std::swap(a.m_val, b.m_val); } +}; + +/** A bitset implementation backed by N integers of type I. */ +template<typename I, unsigned N> +class MultiIntBitSet +{ + // Only binary, unsigned, integer, types allowed. + static_assert(std::is_integral_v<I> && std::is_unsigned_v<I> && std::numeric_limits<I>::radix == 2); + // Cannot be empty. + static_assert(N > 0); + /** The number of bits per integer. */ + static constexpr unsigned LIMB_BITS = std::numeric_limits<I>::digits; + /** Number of elements this set type supports. */ + static constexpr unsigned MAX_SIZE = LIMB_BITS * N; + // No overflow allowed here. + static_assert(MAX_SIZE / LIMB_BITS == N); + /** Array whose member integers store the bits of the set. */ + std::array<I, N> m_val; + /** Dummy type to return using end(). Only used for comparing with Iterator. */ + class IteratorEnd + { + friend class MultiIntBitSet; + constexpr IteratorEnd() = default; + public: + constexpr IteratorEnd(const IteratorEnd&) = default; + }; + /** Iterator type returned by begin(), which efficiently iterates all 1 positions. */ + class Iterator + { + friend class MultiIntBitSet; + const std::array<I, N>* m_ptr; /**< Pointer to array to fetch bits from. */ + I m_val; /**< The remaining bits of (*m_ptr)[m_idx]. */ + unsigned m_pos; /**< The last reported position. */ + unsigned m_idx; /**< The index in *m_ptr currently being iterated over. */ + constexpr Iterator(const std::array<I, N>& ref) noexcept : m_ptr(&ref), m_idx(0) + { + do { + m_val = (*m_ptr)[m_idx]; + if (m_val) { + m_pos = std::countr_zero(m_val) + m_idx * LIMB_BITS; + break; + } + ++m_idx; + } while(m_idx < N); + } + + public: + /** Do not allow external code to construct an Iterator. */ + Iterator() = delete; + // Copying is allowed. + constexpr Iterator(const Iterator&) noexcept = default; + constexpr Iterator& operator=(const Iterator&) noexcept = default; + /** Test whether we are done (can only compare with IteratorEnd). */ + friend constexpr bool operator==(const Iterator& a, const IteratorEnd&) noexcept + { + return a.m_idx == N; + } + /** Progress to the next 1 bit (only if != IteratorEnd). */ + constexpr Iterator& operator++() noexcept + { + Assume(m_idx < N); + m_val &= m_val - I{1U}; + if (m_val == 0) { + while (true) { + ++m_idx; + if (m_idx == N) break; + m_val = (*m_ptr)[m_idx]; + if (m_val) { + m_pos = std::countr_zero(m_val) + m_idx * LIMB_BITS; + break; + } + } + } else { + m_pos = std::countr_zero(m_val) + m_idx * LIMB_BITS; + } + return *this; + } + /** Get the current bit position (only if != IteratorEnd). */ + constexpr unsigned operator*() const noexcept + { + Assume(m_idx < N); + return m_pos; + } + }; + +public: + /** Construct an all-zero bitset. */ + constexpr MultiIntBitSet() noexcept : m_val{} {} + /** Copy construct a bitset. */ + constexpr MultiIntBitSet(const MultiIntBitSet&) noexcept = default; + /** Copy assign a bitset. */ + constexpr MultiIntBitSet& operator=(const MultiIntBitSet&) noexcept = default; + /** Set a bit to 1. */ + void constexpr Set(unsigned pos) noexcept + { + Assume(pos < MAX_SIZE); + m_val[pos / LIMB_BITS] |= I{1U} << (pos % LIMB_BITS); + } + /** Set a bit to the specified value. */ + void constexpr Set(unsigned pos, bool val) noexcept + { + Assume(pos < MAX_SIZE); + m_val[pos / LIMB_BITS] = (m_val[pos / LIMB_BITS] & ~I(I{1U} << (pos % LIMB_BITS))) | + (I{val} << (pos % LIMB_BITS)); + } + /** Construct a bitset from a list of values. */ + constexpr MultiIntBitSet(std::initializer_list<unsigned> ilist) noexcept : m_val{} + { + for (auto pos : ilist) Set(pos); + } + /** Set a bitset to a list of values. */ + constexpr MultiIntBitSet& operator=(std::initializer_list<unsigned> ilist) noexcept + { + m_val.fill(0); + for (auto pos : ilist) Set(pos); + return *this; + } + /** Set a bit to 0. */ + void constexpr Reset(unsigned pos) noexcept + { + Assume(pos < MAX_SIZE); + m_val[pos / LIMB_BITS] &= ~I(I{1U} << (pos % LIMB_BITS)); + } + /** Retrieve a bit at the given position. */ + bool constexpr operator[](unsigned pos) const noexcept + { + Assume(pos < MAX_SIZE); + return (m_val[pos / LIMB_BITS] >> (pos % LIMB_BITS)) & 1U; + } + /** Construct a bitset with the singleton pos. */ + static constexpr MultiIntBitSet Singleton(unsigned pos) noexcept + { + Assume(pos < MAX_SIZE); + MultiIntBitSet ret; + ret.m_val[pos / LIMB_BITS] = I{1U} << (pos % LIMB_BITS); + return ret; + } + /** Construct a bitset with bits 0..count-1 (inclusive) set to 1. */ + static constexpr MultiIntBitSet Fill(unsigned count) noexcept + { + Assume(count <= MAX_SIZE); + MultiIntBitSet ret; + if (count) { + unsigned i = 0; + while (count > LIMB_BITS) { + ret.m_val[i++] = ~I{0}; + count -= LIMB_BITS; + } + ret.m_val[i] = I(~I{0}) >> (LIMB_BITS - count); + } + return ret; + } + /** Return the number of bits that this object holds. */ + static constexpr unsigned Size() noexcept { return MAX_SIZE; } + /** Compute the number of 1 bits in the bitset. */ + unsigned constexpr Count() const noexcept + { + unsigned ret{0}; + for (I v : m_val) ret += PopCount(v); + return ret; + } + /** Check if all bits are 0. */ + bool constexpr None() const noexcept + { + for (auto v : m_val) { + if (v != 0) return false; + } + return true; + } + /** Check if any bits are 1. */ + bool constexpr Any() const noexcept { return !None(); } + /** Return an object that iterates over all 1 bits (++ and * only allowed when != end()). */ + Iterator constexpr begin() const noexcept { return Iterator(m_val); } + /** Return a dummy object to compare Iterators with. */ + IteratorEnd constexpr end() const noexcept { return IteratorEnd(); } + /** Find the first element (requires Any()). */ + unsigned constexpr First() const noexcept + { + unsigned p = 0; + while (m_val[p] == 0) { + ++p; + Assume(p < N); + } + return std::countr_zero(m_val[p]) + p * LIMB_BITS; + } + /** Find the last element (requires Any()). */ + unsigned constexpr Last() const noexcept + { + unsigned p = N - 1; + while (m_val[p] == 0) { + Assume(p > 0); + --p; + } + return std::bit_width(m_val[p]) - 1 + p * LIMB_BITS; + } + /** Set this object's bits to be the binary OR between respective bits from this and a. */ + constexpr MultiIntBitSet& operator|=(const MultiIntBitSet& a) noexcept + { + for (unsigned i = 0; i < N; ++i) { + m_val[i] |= a.m_val[i]; + } + return *this; + } + /** Set this object's bits to be the binary AND between respective bits from this and a. */ + constexpr MultiIntBitSet& operator&=(const MultiIntBitSet& a) noexcept + { + for (unsigned i = 0; i < N; ++i) { + m_val[i] &= a.m_val[i]; + } + return *this; + } + /** Set this object's bits to be the binary AND NOT between respective bits from this and a. */ + constexpr MultiIntBitSet& operator-=(const MultiIntBitSet& a) noexcept + { + for (unsigned i = 0; i < N; ++i) { + m_val[i] &= ~a.m_val[i]; + } + return *this; + } + /** Set this object's bits to be the binary XOR between respective bits from this and a. */ + constexpr MultiIntBitSet& operator^=(const MultiIntBitSet& a) noexcept + { + for (unsigned i = 0; i < N; ++i) { + m_val[i] ^= a.m_val[i]; + } + return *this; + } + /** Check whether the intersection between two sets is non-empty. */ + constexpr bool Overlaps(const MultiIntBitSet& a) const noexcept + { + for (unsigned i = 0; i < N; ++i) { + if (m_val[i] & a.m_val[i]) return true; + } + return false; + } + /** Return an object with the binary AND between respective bits from a and b. */ + friend constexpr MultiIntBitSet operator&(const MultiIntBitSet& a, const MultiIntBitSet& b) noexcept + { + MultiIntBitSet r; + for (unsigned i = 0; i < N; ++i) { + r.m_val[i] = a.m_val[i] & b.m_val[i]; + } + return r; + } + /** Return an object with the binary OR between respective bits from a and b. */ + friend constexpr MultiIntBitSet operator|(const MultiIntBitSet& a, const MultiIntBitSet& b) noexcept + { + MultiIntBitSet r; + for (unsigned i = 0; i < N; ++i) { + r.m_val[i] = a.m_val[i] | b.m_val[i]; + } + return r; + } + /** Return an object with the binary AND NOT between respective bits from a and b. */ + friend constexpr MultiIntBitSet operator-(const MultiIntBitSet& a, const MultiIntBitSet& b) noexcept + { + MultiIntBitSet r; + for (unsigned i = 0; i < N; ++i) { + r.m_val[i] = a.m_val[i] & ~b.m_val[i]; + } + return r; + } + /** Return an object with the binary XOR between respective bits from a and b. */ + friend constexpr MultiIntBitSet operator^(const MultiIntBitSet& a, const MultiIntBitSet& b) noexcept + { + MultiIntBitSet r; + for (unsigned i = 0; i < N; ++i) { + r.m_val[i] = a.m_val[i] ^ b.m_val[i]; + } + return r; + } + /** Check if bitset a is a superset of bitset b (= every 1 bit in b is also in a). */ + constexpr bool IsSupersetOf(const MultiIntBitSet& a) const noexcept + { + for (unsigned i = 0; i < N; ++i) { + if (a.m_val[i] & ~m_val[i]) return false; + } + return true; + } + /** Check if bitset a is a subset of bitset b (= every 1 bit in a is also in b). */ + constexpr bool IsSubsetOf(const MultiIntBitSet& a) const noexcept + { + for (unsigned i = 0; i < N; ++i) { + if (m_val[i] & ~a.m_val[i]) return false; + } + return true; + } + /** Check if bitset a and bitset b are identical. */ + friend constexpr bool operator==(const MultiIntBitSet& a, const MultiIntBitSet& b) noexcept = default; + /** Swap two bitsets. */ + friend constexpr void swap(MultiIntBitSet& a, MultiIntBitSet& b) noexcept { std::swap(a.m_val, b.m_val); } +}; + +} // namespace bitset_detail + +// BitSet dispatches to IntBitSet or MultiIntBitSet as appropriate for the requested minimum number +// of bits. Use IntBitSet up to 32-bit, or up to 64-bit on 64-bit platforms; above that, use a +// MultiIntBitSet of size_t. +template<unsigned BITS> +using BitSet = std::conditional_t<(BITS <= 32), bitset_detail::IntBitSet<uint32_t>, + std::conditional_t<(BITS <= std::numeric_limits<size_t>::digits), bitset_detail::IntBitSet<size_t>, + bitset_detail::MultiIntBitSet<size_t, (BITS + std::numeric_limits<size_t>::digits - 1) / std::numeric_limits<size_t>::digits>>>; + +#endif // BITCOIN_UTIL_BITSET_H diff --git a/src/util/error.cpp b/src/util/error.cpp deleted file mode 100644 index 309877d067..0000000000 --- a/src/util/error.cpp +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) 2010-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <util/error.h> - -#include <tinyformat.h> -#include <util/translation.h> - -#include <cassert> -#include <string> - -bilingual_str TransactionErrorString(const TransactionError err) -{ - switch (err) { - case TransactionError::OK: - return Untranslated("No error"); - case TransactionError::MISSING_INPUTS: - return Untranslated("Inputs missing or spent"); - case TransactionError::ALREADY_IN_CHAIN: - return Untranslated("Transaction already in block chain"); - case TransactionError::P2P_DISABLED: - return Untranslated("Peer-to-peer functionality missing or disabled"); - case TransactionError::MEMPOOL_REJECTED: - return Untranslated("Transaction rejected by mempool"); - case TransactionError::MEMPOOL_ERROR: - return Untranslated("Mempool internal error"); - case TransactionError::INVALID_PSBT: - return Untranslated("PSBT is not well-formed"); - case TransactionError::PSBT_MISMATCH: - return Untranslated("PSBTs not compatible (different transactions)"); - case TransactionError::SIGHASH_MISMATCH: - return Untranslated("Specified sighash value does not match value stored in PSBT"); - case TransactionError::MAX_FEE_EXCEEDED: - return Untranslated("Fee exceeds maximum configured by user (e.g. -maxtxfee, maxfeerate)"); - case TransactionError::MAX_BURN_EXCEEDED: - return Untranslated("Unspendable output exceeds maximum configured by user (maxburnamount)"); - case TransactionError::EXTERNAL_SIGNER_NOT_FOUND: - return Untranslated("External signer not found"); - case TransactionError::EXTERNAL_SIGNER_FAILED: - return Untranslated("External signer failed to sign"); - case TransactionError::INVALID_PACKAGE: - return Untranslated("Transaction rejected due to invalid package"); - // no default case, so the compiler can warn about missing cases - } - assert(false); -} - -bilingual_str ResolveErrMsg(const std::string& optname, const std::string& strBind) -{ - return strprintf(_("Cannot resolve -%s address: '%s'"), optname, strBind); -} - -bilingual_str InvalidPortErrMsg(const std::string& optname, const std::string& invalid_value) -{ - return strprintf(_("Invalid port specified in %s: '%s'"), optname, invalid_value); -} - -bilingual_str AmountHighWarn(const std::string& optname) -{ - return strprintf(_("%s is set very high!"), optname); -} - -bilingual_str AmountErrMsg(const std::string& optname, const std::string& strValue) -{ - return strprintf(_("Invalid amount for -%s=<amount>: '%s'"), optname, strValue); -} diff --git a/src/util/error.h b/src/util/error.h deleted file mode 100644 index a52a8f47de..0000000000 --- a/src/util/error.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) 2010-2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_UTIL_ERROR_H -#define BITCOIN_UTIL_ERROR_H - -/** - * util/error.h is a common place for definitions of simple error types and - * string functions. Types and functions defined here should not require any - * outside dependencies. - * - * Error types defined here can be used in different parts of the - * codebase, to avoid the need to write boilerplate code catching and - * translating errors passed across wallet/node/rpc/gui code boundaries. - */ - -#include <string> - -struct bilingual_str; - -enum class TransactionError { - OK, //!< No error - MISSING_INPUTS, - ALREADY_IN_CHAIN, - P2P_DISABLED, - MEMPOOL_REJECTED, - MEMPOOL_ERROR, - INVALID_PSBT, - PSBT_MISMATCH, - SIGHASH_MISMATCH, - MAX_FEE_EXCEEDED, - MAX_BURN_EXCEEDED, - EXTERNAL_SIGNER_NOT_FOUND, - EXTERNAL_SIGNER_FAILED, - INVALID_PACKAGE, -}; - -bilingual_str TransactionErrorString(const TransactionError error); - -bilingual_str ResolveErrMsg(const std::string& optname, const std::string& strBind); - -bilingual_str InvalidPortErrMsg(const std::string& optname, const std::string& strPort); - -bilingual_str AmountHighWarn(const std::string& optname); - -bilingual_str AmountErrMsg(const std::string& optname, const std::string& strValue); - -#endif // BITCOIN_UTIL_ERROR_H diff --git a/src/util/fees.cpp b/src/util/fees.cpp deleted file mode 100644 index 8ada02ce54..0000000000 --- a/src/util/fees.cpp +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2020 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <util/fees.h> - -#include <policy/fees.h> -#include <util/strencodings.h> -#include <util/string.h> - -#include <map> -#include <string> -#include <vector> -#include <utility> - -std::string StringForFeeReason(FeeReason reason) -{ - static const std::map<FeeReason, std::string> fee_reason_strings = { - {FeeReason::NONE, "None"}, - {FeeReason::HALF_ESTIMATE, "Half Target 60% Threshold"}, - {FeeReason::FULL_ESTIMATE, "Target 85% Threshold"}, - {FeeReason::DOUBLE_ESTIMATE, "Double Target 95% Threshold"}, - {FeeReason::CONSERVATIVE, "Conservative Double Target longer horizon"}, - {FeeReason::MEMPOOL_MIN, "Mempool Min Fee"}, - {FeeReason::PAYTXFEE, "PayTxFee set"}, - {FeeReason::FALLBACK, "Fallback fee"}, - {FeeReason::REQUIRED, "Minimum Required Fee"}, - }; - auto reason_string = fee_reason_strings.find(reason); - - if (reason_string == fee_reason_strings.end()) return "Unknown"; - - return reason_string->second; -} - -const std::vector<std::pair<std::string, FeeEstimateMode>>& FeeModeMap() -{ - static const std::vector<std::pair<std::string, FeeEstimateMode>> FEE_MODES = { - {"unset", FeeEstimateMode::UNSET}, - {"economical", FeeEstimateMode::ECONOMICAL}, - {"conservative", FeeEstimateMode::CONSERVATIVE}, - }; - return FEE_MODES; -} - -std::string FeeModes(const std::string& delimiter) -{ - return Join(FeeModeMap(), delimiter, [&](const std::pair<std::string, FeeEstimateMode>& i) { return i.first; }); -} - -std::string InvalidEstimateModeErrorMessage() -{ - return "Invalid estimate_mode parameter, must be one of: \"" + FeeModes("\", \"") + "\""; -} - -bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode) -{ - auto searchkey = ToUpper(mode_string); - for (const auto& pair : FeeModeMap()) { - if (ToUpper(pair.first) == searchkey) { - fee_estimate_mode = pair.second; - return true; - } - } - return false; -} diff --git a/src/util/fees.h b/src/util/fees.h deleted file mode 100644 index 10ba1e4f85..0000000000 --- a/src/util/fees.h +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2020 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_UTIL_FEES_H -#define BITCOIN_UTIL_FEES_H - -#include <string> - -enum class FeeEstimateMode; -enum class FeeReason; - -bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode); -std::string StringForFeeReason(FeeReason reason); -std::string FeeModes(const std::string& delimiter); -std::string InvalidEstimateModeErrorMessage(); - -#endif // BITCOIN_UTIL_FEES_H diff --git a/src/util/moneystr.cpp b/src/util/moneystr.cpp index 9181329afc..1ed3b2ac96 100644 --- a/src/util/moneystr.cpp +++ b/src/util/moneystr.cpp @@ -13,6 +13,9 @@ #include <cstdint> #include <optional> +using util::ContainsNoNUL; +using util::TrimString; + std::string FormatMoney(const CAmount n) { // Note: not using straight sprintf here because we do NOT want diff --git a/src/util/spanparsing.h b/src/util/spanparsing.h deleted file mode 100644 index 765fe13aca..0000000000 --- a/src/util/spanparsing.h +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (c) 2018-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_UTIL_SPANPARSING_H -#define BITCOIN_UTIL_SPANPARSING_H - -#include <span.h> - -#include <string> -#include <string_view> -#include <vector> - -namespace spanparsing { - -/** Parse a constant. - * - * If sp's initial part matches str, sp is updated to skip that part, and true is returned. - * Otherwise sp is unmodified and false is returned. - */ -bool Const(const std::string& str, Span<const char>& sp); - -/** Parse a function call. - * - * If sp's initial part matches str + "(", and sp ends with ")", sp is updated to be the - * section between the braces, and true is returned. Otherwise sp is unmodified and false - * is returned. - */ -bool Func(const std::string& str, Span<const char>& sp); - -/** Extract the expression that sp begins with. - * - * This function will return the initial part of sp, up to (but not including) the first - * comma or closing brace, skipping ones that are surrounded by braces. So for example, - * for "foo(bar(1),2),3" the initial part "foo(bar(1),2)" will be returned. sp will be - * updated to skip the initial part that is returned. - */ -Span<const char> Expr(Span<const char>& sp); - -/** Split a string on any char found in separators, returning a vector. - * - * If sep does not occur in sp, a singleton with the entirety of sp is returned. - * - * Note that this function does not care about braces, so splitting - * "foo(bar(1),2),3) on ',' will return {"foo(bar(1)", "2)", "3)"}. - */ -template <typename T = Span<const char>> -std::vector<T> Split(const Span<const char>& sp, std::string_view separators) -{ - std::vector<T> ret; - auto it = sp.begin(); - auto start = it; - while (it != sp.end()) { - if (separators.find(*it) != std::string::npos) { - ret.emplace_back(start, it); - start = it + 1; - } - ++it; - } - ret.emplace_back(start, it); - return ret; -} - -/** Split a string on every instance of sep, returning a vector. - * - * If sep does not occur in sp, a singleton with the entirety of sp is returned. - * - * Note that this function does not care about braces, so splitting - * "foo(bar(1),2),3) on ',' will return {"foo(bar(1)", "2)", "3)"}. - */ -template <typename T = Span<const char>> -std::vector<T> Split(const Span<const char>& sp, char sep) -{ - return Split<T>(sp, std::string_view{&sep, 1}); -} - -} // namespace spanparsing - -#endif // BITCOIN_UTIL_SPANPARSING_H diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index 7b5ded2975..e030262a32 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -3,9 +3,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <span.h> #include <util/strencodings.h> +#include <crypto/hex_base.h> +#include <span.h> + #include <array> #include <cassert> #include <cstring> @@ -36,29 +38,6 @@ std::string SanitizeString(std::string_view str, int rule) return result; } -const signed char p_util_hexdigit[256] = -{ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - 0,1,2,3,4,5,6,7,8,9,-1,-1,-1,-1,-1,-1, - -1,0xa,0xb,0xc,0xd,0xe,0xf,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,0xa,0xb,0xc,0xd,0xe,0xf,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, }; - -signed char HexDigit(char c) -{ - return p_util_hexdigit[(unsigned char)c]; -} - bool IsHex(std::string_view str) { for (char c : str) { @@ -466,40 +445,6 @@ std::string Capitalize(std::string str) return str; } -namespace { - -using ByteAsHex = std::array<char, 2>; - -constexpr std::array<ByteAsHex, 256> CreateByteToHexMap() -{ - constexpr char hexmap[16] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; - - std::array<ByteAsHex, 256> byte_to_hex{}; - for (size_t i = 0; i < byte_to_hex.size(); ++i) { - byte_to_hex[i][0] = hexmap[i >> 4]; - byte_to_hex[i][1] = hexmap[i & 15]; - } - return byte_to_hex; -} - -} // namespace - -std::string HexStr(const Span<const uint8_t> s) -{ - std::string rv(s.size() * 2, '\0'); - static constexpr auto byte_to_hex = CreateByteToHexMap(); - static_assert(sizeof(byte_to_hex) == 512); - - char* it = rv.data(); - for (uint8_t v : s) { - std::memcpy(it, byte_to_hex[v].data(), 2); - it += 2; - } - - assert(it == rv.data() + rv.size()); - return rv; -} - std::optional<uint64_t> ParseByteUnits(std::string_view str, ByteUnit default_multiplier) { if (str.empty()) { diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 439678c24a..e5c2d3ddf2 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -9,6 +9,7 @@ #ifndef BITCOIN_UTIL_STRENCODINGS_H #define BITCOIN_UTIL_STRENCODINGS_H +#include <crypto/hex_base.h> // IWYU pragma: export #include <span.h> #include <util/string.h> @@ -66,7 +67,6 @@ std::vector<Byte> ParseHex(std::string_view hex_str) { return TryParseHex<Byte>(hex_str).value_or(std::vector<Byte>{}); } -signed char HexDigit(char c); /* Returns true if each character in str is a hex character, and has an even * number of hex digits.*/ bool IsHex(std::string_view str); @@ -122,7 +122,7 @@ T LocaleIndependentAtoi(std::string_view str) static_assert(std::is_integral<T>::value); T result; // Emulate atoi(...) handling of white space and leading +/-. - std::string_view s = TrimStringView(str); + std::string_view s = util::TrimStringView(str); if (!s.empty() && s[0] == '+') { if (s.length() >= 2 && s[1] == '-') { return 0; @@ -232,13 +232,6 @@ std::optional<T> ToIntegral(std::string_view str) [[nodiscard]] bool ParseUInt64(std::string_view str, uint64_t *out); /** - * Convert a span of bytes to a lower-case hexadecimal string. - */ -std::string HexStr(const Span<const uint8_t> s); -inline std::string HexStr(const Span<const char> s) { return HexStr(MakeUCharSpan(s)); } -inline std::string HexStr(const Span<const std::byte> s) { return HexStr(MakeUCharSpan(s)); } - -/** * Format a paragraph of text to a fixed width, adding spaces for * indentation to any added line. */ diff --git a/src/util/string.cpp b/src/util/string.cpp index 3d31849745..47c6b74d4f 100644 --- a/src/util/string.cpp +++ b/src/util/string.cpp @@ -7,8 +7,10 @@ #include <regex> #include <string> +namespace util { void ReplaceAll(std::string& in_out, const std::string& search, const std::string& substitute) { if (search.empty()) return; in_out = std::regex_replace(in_out, std::regex(search), substitute); } +} // namespace util diff --git a/src/util/string.h b/src/util/string.h index dab92942fb..e2e470f4ab 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_UTIL_STRING_H #define BITCOIN_UTIL_STRING_H -#include <util/spanparsing.h> +#include <span.h> #include <array> #include <cstdint> @@ -16,16 +16,54 @@ #include <string_view> // IWYU pragma: export #include <vector> +namespace util { void ReplaceAll(std::string& in_out, const std::string& search, const std::string& substitute); +/** Split a string on any char found in separators, returning a vector. + * + * If sep does not occur in sp, a singleton with the entirety of sp is returned. + * + * Note that this function does not care about braces, so splitting + * "foo(bar(1),2),3) on ',' will return {"foo(bar(1)", "2)", "3)"}. + */ +template <typename T = Span<const char>> +std::vector<T> Split(const Span<const char>& sp, std::string_view separators) +{ + std::vector<T> ret; + auto it = sp.begin(); + auto start = it; + while (it != sp.end()) { + if (separators.find(*it) != std::string::npos) { + ret.emplace_back(start, it); + start = it + 1; + } + ++it; + } + ret.emplace_back(start, it); + return ret; +} + +/** Split a string on every instance of sep, returning a vector. + * + * If sep does not occur in sp, a singleton with the entirety of sp is returned. + * + * Note that this function does not care about braces, so splitting + * "foo(bar(1),2),3) on ',' will return {"foo(bar(1)", "2)", "3)"}. + */ +template <typename T = Span<const char>> +std::vector<T> Split(const Span<const char>& sp, char sep) +{ + return Split<T>(sp, std::string_view{&sep, 1}); +} + [[nodiscard]] inline std::vector<std::string> SplitString(std::string_view str, char sep) { - return spanparsing::Split<std::string>(str, sep); + return Split<std::string>(str, sep); } [[nodiscard]] inline std::vector<std::string> SplitString(std::string_view str, std::string_view separators) { - return spanparsing::Split<std::string>(str, separators); + return Split<std::string>(str, separators); } [[nodiscard]] inline std::string_view TrimStringView(std::string_view str, std::string_view pattern = " \f\n\r\t\v") @@ -125,5 +163,6 @@ template <typename T1, size_t PREFIX_LEN> return obj.size() >= PREFIX_LEN && std::equal(std::begin(prefix), std::end(prefix), std::begin(obj)); } +} // namespace util #endif // BITCOIN_UTIL_STRING_H diff --git a/src/util/vecdeque.h b/src/util/vecdeque.h new file mode 100644 index 0000000000..a9264a5ad6 --- /dev/null +++ b/src/util/vecdeque.h @@ -0,0 +1,317 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_VECDEQUE_H +#define BITCOIN_UTIL_VECDEQUE_H + +#include <util/check.h> + +#include <cstring> +#include <memory> +#include <type_traits> + +/** Data structure largely mimicking std::deque, but using single preallocated ring buffer. + * + * - More efficient and better memory locality than std::deque. + * - Most operations ({push_,pop_,emplace_,}{front,back}(), operator[], ...) are O(1), + * unless reallocation is needed (in which case they are O(n)). + * - Supports reserve(), capacity(), shrink_to_fit() like vectors. + * - No iterator support. + * - Data is not stored in a single contiguous block, so no data(). + */ +template<typename T> +class VecDeque +{ + /** Pointer to allocated memory. Can contain constructed and uninitialized T objects. */ + T* m_buffer{nullptr}; + /** m_buffer + m_offset points to first object in queue. m_offset = 0 if m_capacity is 0; + * otherwise 0 <= m_offset < m_capacity. */ + size_t m_offset{0}; + /** Number of objects in the container. 0 <= m_size <= m_capacity. */ + size_t m_size{0}; + /** The size of m_buffer, expressed as a multiple of the size of T. */ + size_t m_capacity{0}; + + /** Returns the number of populated objects between m_offset and the end of the buffer. */ + size_t FirstPart() const noexcept { return std::min(m_capacity - m_offset, m_size); } + + void Reallocate(size_t capacity) + { + Assume(capacity >= m_size); + Assume((m_offset == 0 && m_capacity == 0) || m_offset < m_capacity); + // Allocate new buffer. + T* new_buffer = capacity ? std::allocator<T>().allocate(capacity) : nullptr; + if (capacity) { + if constexpr (std::is_trivially_copyable_v<T>) { + // When T is trivially copyable, just copy the data over from old to new buffer. + size_t first_part = FirstPart(); + if (first_part != 0) { + std::memcpy(new_buffer, m_buffer + m_offset, first_part * sizeof(T)); + } + if (first_part != m_size) { + std::memcpy(new_buffer + first_part, m_buffer, (m_size - first_part) * sizeof(T)); + } + } else { + // Otherwise move-construct in place in the new buffer, and destroy old buffer objects. + size_t old_pos = m_offset; + for (size_t new_pos = 0; new_pos < m_size; ++new_pos) { + std::construct_at(new_buffer + new_pos, std::move(*(m_buffer + old_pos))); + std::destroy_at(m_buffer + old_pos); + ++old_pos; + if (old_pos == m_capacity) old_pos = 0; + } + } + } + // Deallocate old buffer and update housekeeping. + std::allocator<T>().deallocate(m_buffer, m_capacity); + m_buffer = new_buffer; + m_offset = 0; + m_capacity = capacity; + Assume((m_offset == 0 && m_capacity == 0) || m_offset < m_capacity); + } + + /** What index in the buffer does logical entry number pos have? */ + size_t BufferIndex(size_t pos) const noexcept + { + Assume(pos < m_capacity); + // The expression below is used instead of the more obvious (pos + m_offset >= m_capacity), + // because the addition there could in theory overflow with very large deques. + if (pos >= m_capacity - m_offset) { + return (m_offset + pos) - m_capacity; + } else { + return m_offset + pos; + } + } + + /** Specialization of resize() that can only shrink. Separate so that clear() can call it + * without requiring a default T constructor. */ + void ResizeDown(size_t size) noexcept + { + Assume(size <= m_size); + if constexpr (std::is_trivially_destructible_v<T>) { + // If T is trivially destructible, we do not need to do anything but update the + // housekeeping record. Default constructor or zero-filling will be used when + // the space is reused. + m_size = size; + } else { + // If not, we need to invoke the destructor for every element separately. + while (m_size > size) { + std::destroy_at(m_buffer + BufferIndex(m_size - 1)); + --m_size; + } + } + } + +public: + VecDeque() noexcept = default; + + /** Resize the deque to be exactly size size (adding default-constructed elements if needed). */ + void resize(size_t size) + { + if (size < m_size) { + // Delegate to ResizeDown when shrinking. + ResizeDown(size); + } else if (size > m_size) { + // When growing, first see if we need to allocate more space. + if (size > m_capacity) Reallocate(size); + while (m_size < size) { + std::construct_at(m_buffer + BufferIndex(m_size)); + ++m_size; + } + } + } + + /** Resize the deque to be size 0. The capacity will remain unchanged. */ + void clear() noexcept { ResizeDown(0); } + + /** Destroy a deque. */ + ~VecDeque() + { + clear(); + Reallocate(0); + } + + /** Copy-assign a deque. */ + VecDeque& operator=(const VecDeque& other) + { + if (&other == this) [[unlikely]] return *this; + clear(); + Reallocate(other.m_size); + if constexpr (std::is_trivially_copyable_v<T>) { + size_t first_part = other.FirstPart(); + Assume(first_part > 0 || m_size == 0); + if (first_part != 0) { + std::memcpy(m_buffer, other.m_buffer + other.m_offset, first_part * sizeof(T)); + } + if (first_part != other.m_size) { + std::memcpy(m_buffer + first_part, other.m_buffer, (other.m_size - first_part) * sizeof(T)); + } + m_size = other.m_size; + } else { + while (m_size < other.m_size) { + std::construct_at(m_buffer + BufferIndex(m_size), other[m_size]); + ++m_size; + } + } + return *this; + } + + /** Swap two deques. */ + void swap(VecDeque& other) noexcept + { + std::swap(m_buffer, other.m_buffer); + std::swap(m_offset, other.m_offset); + std::swap(m_size, other.m_size); + std::swap(m_capacity, other.m_capacity); + } + + /** Non-member version of swap. */ + friend void swap(VecDeque& a, VecDeque& b) noexcept { a.swap(b); } + + /** Move-assign a deque. */ + VecDeque& operator=(VecDeque&& other) noexcept + { + swap(other); + return *this; + } + + /** Copy-construct a deque. */ + VecDeque(const VecDeque& other) { *this = other; } + /** Move-construct a deque. */ + VecDeque(VecDeque&& other) noexcept { swap(other); } + + /** Equality comparison between two deques (only compares size+contents, not capacity). */ + bool friend operator==(const VecDeque& a, const VecDeque& b) + { + if (a.m_size != b.m_size) return false; + for (size_t i = 0; i < a.m_size; ++i) { + if (a[i] != b[i]) return false; + } + return true; + } + + /** Comparison between two deques, implementing lexicographic ordering on the contents. */ + std::strong_ordering friend operator<=>(const VecDeque& a, const VecDeque& b) + { + size_t pos_a{0}, pos_b{0}; + while (pos_a < a.m_size && pos_b < b.m_size) { + auto cmp = a[pos_a++] <=> b[pos_b++]; + if (cmp != 0) return cmp; + } + return a.m_size <=> b.m_size; + } + + /** Increase the capacity to capacity. Capacity will not shrink. */ + void reserve(size_t capacity) + { + if (capacity > m_capacity) Reallocate(capacity); + } + + /** Make the capacity equal to the size. The contents does not change. */ + void shrink_to_fit() + { + if (m_capacity > m_size) Reallocate(m_size); + } + + /** Construct a new element at the end of the deque. */ + template<typename... Args> + void emplace_back(Args&&... args) + { + if (m_size == m_capacity) Reallocate((m_size + 1) * 2); + std::construct_at(m_buffer + BufferIndex(m_size), std::forward<Args>(args)...); + ++m_size; + } + + /** Move-construct a new element at the end of the deque. */ + void push_back(T&& elem) { emplace_back(std::move(elem)); } + + /** Copy-construct a new element at the end of the deque. */ + void push_back(const T& elem) { emplace_back(elem); } + + /** Construct a new element at the beginning of the deque. */ + template<typename... Args> + void emplace_front(Args&&... args) + { + if (m_size == m_capacity) Reallocate((m_size + 1) * 2); + std::construct_at(m_buffer + BufferIndex(m_capacity - 1), std::forward<Args>(args)...); + if (m_offset == 0) m_offset = m_capacity; + --m_offset; + ++m_size; + } + + /** Copy-construct a new element at the beginning of the deque. */ + void push_front(const T& elem) { emplace_front(elem); } + + /** Move-construct a new element at the beginning of the deque. */ + void push_front(T&& elem) { emplace_front(std::move(elem)); } + + /** Remove the first element of the deque. Requires !empty(). */ + void pop_front() + { + Assume(m_size); + std::destroy_at(m_buffer + m_offset); + --m_size; + ++m_offset; + if (m_offset == m_capacity) m_offset = 0; + } + + /** Remove the last element of the deque. Requires !empty(). */ + void pop_back() + { + Assume(m_size); + std::destroy_at(m_buffer + BufferIndex(m_size - 1)); + --m_size; + } + + /** Get a mutable reference to the first element of the deque. Requires !empty(). */ + T& front() noexcept + { + Assume(m_size); + return m_buffer[m_offset]; + } + + /** Get a const reference to the first element of the deque. Requires !empty(). */ + const T& front() const noexcept + { + Assume(m_size); + return m_buffer[m_offset]; + } + + /** Get a mutable reference to the last element of the deque. Requires !empty(). */ + T& back() noexcept + { + Assume(m_size); + return m_buffer[BufferIndex(m_size - 1)]; + } + + /** Get a const reference to the last element of the deque. Requires !empty(). */ + const T& back() const noexcept + { + Assume(m_size); + return m_buffer[BufferIndex(m_size - 1)]; + } + + /** Get a mutable reference to the element in the deque at the given index. Requires idx < size(). */ + T& operator[](size_t idx) noexcept + { + Assume(idx < m_size); + return m_buffer[BufferIndex(idx)]; + } + + /** Get a const reference to the element in the deque at the given index. Requires idx < size(). */ + const T& operator[](size_t idx) const noexcept + { + Assume(idx < m_size); + return m_buffer[BufferIndex(idx)]; + } + + /** Test whether the contents of this deque is empty. */ + bool empty() const noexcept { return m_size == 0; } + /** Get the number of elements in this deque. */ + size_t size() const noexcept { return m_size; } + /** Get the capacity of this deque (maximum size it can have without reallocating). */ + size_t capacity() const noexcept { return m_capacity; } +}; + +#endif // BITCOIN_UTIL_VECDEQUE_H diff --git a/src/validation.cpp b/src/validation.cpp index d398ec7406..3e9ba08bb1 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -27,14 +27,15 @@ #include <kernel/mempool_entry.h> #include <kernel/messagestartchars.h> #include <kernel/notifications_interface.h> +#include <kernel/warning.h> #include <logging.h> #include <logging/timer.h> #include <node/blockstorage.h> #include <node/utxo_snapshot.h> -#include <policy/v3_policy.h> #include <policy/policy.h> #include <policy/rbf.h> #include <policy/settings.h> +#include <policy/v3_policy.h> #include <pow.h> #include <primitives/block.h> #include <primitives/transaction.h> @@ -57,11 +58,11 @@ #include <util/result.h> #include <util/signalinterrupt.h> #include <util/strencodings.h> +#include <util/string.h> #include <util/time.h> #include <util/trace.h> #include <util/translation.h> #include <validationinterface.h> -#include <warnings.h> #include <algorithm> #include <cassert> @@ -455,11 +456,14 @@ public: * the mempool. */ std::vector<COutPoint>& m_coins_to_uncache; + /** When true, the transaction or package will not be submitted to the mempool. */ const bool m_test_accept; - /** Whether we allow transactions to replace mempool transactions by BIP125 rules. If false, + /** Whether we allow transactions to replace mempool transactions. If false, * any transaction spending the same inputs as a transaction in the mempool is considered * a conflict. */ const bool m_allow_replacement; + /** When true, allow sibling eviction. This only occurs in single transaction package settings. */ + const bool m_allow_sibling_eviction; /** When true, the mempool will not be trimmed when any transactions are submitted in * Finalize(). Instead, limits should be enforced at the end to ensure the package is not * partially submitted. @@ -475,6 +479,9 @@ public: */ const std::optional<CFeeRate> m_client_maxfeerate; + /** Whether CPFP carveout and RBF carveout are granted. */ + const bool m_allow_carveouts; + /** Parameters for single transaction mempool validation. */ static ATMPArgs SingleAccept(const CChainParams& chainparams, int64_t accept_time, bool bypass_limits, std::vector<COutPoint>& coins_to_uncache, @@ -485,9 +492,11 @@ public: /* m_coins_to_uncache */ coins_to_uncache, /* m_test_accept */ test_accept, /* m_allow_replacement */ true, + /* m_allow_sibling_eviction */ true, /* m_package_submission */ false, /* m_package_feerates */ false, /* m_client_maxfeerate */ {}, // checked by caller + /* m_allow_carveouts */ true, }; } @@ -500,9 +509,11 @@ public: /* m_coins_to_uncache */ coins_to_uncache, /* m_test_accept */ true, /* m_allow_replacement */ false, + /* m_allow_sibling_eviction */ false, /* m_package_submission */ false, // not submitting to mempool /* m_package_feerates */ false, /* m_client_maxfeerate */ {}, // checked by caller + /* m_allow_carveouts */ false, }; } @@ -514,10 +525,12 @@ public: /* m_bypass_limits */ false, /* m_coins_to_uncache */ coins_to_uncache, /* m_test_accept */ false, - /* m_allow_replacement */ false, + /* m_allow_replacement */ true, + /* m_allow_sibling_eviction */ false, /* m_package_submission */ true, /* m_package_feerates */ true, /* m_client_maxfeerate */ client_maxfeerate, + /* m_allow_carveouts */ false, }; } @@ -529,9 +542,11 @@ public: /* m_coins_to_uncache */ package_args.m_coins_to_uncache, /* m_test_accept */ package_args.m_test_accept, /* m_allow_replacement */ true, + /* m_allow_sibling_eviction */ true, /* m_package_submission */ true, // do not LimitMempoolSize in Finalize() /* m_package_feerates */ false, // only 1 transaction /* m_client_maxfeerate */ package_args.m_client_maxfeerate, + /* m_allow_carveouts */ false, }; } @@ -544,19 +559,31 @@ public: std::vector<COutPoint>& coins_to_uncache, bool test_accept, bool allow_replacement, + bool allow_sibling_eviction, bool package_submission, bool package_feerates, - std::optional<CFeeRate> client_maxfeerate) + std::optional<CFeeRate> client_maxfeerate, + bool allow_carveouts) : m_chainparams{chainparams}, m_accept_time{accept_time}, m_bypass_limits{bypass_limits}, m_coins_to_uncache{coins_to_uncache}, m_test_accept{test_accept}, m_allow_replacement{allow_replacement}, + m_allow_sibling_eviction{allow_sibling_eviction}, m_package_submission{package_submission}, m_package_feerates{package_feerates}, - m_client_maxfeerate{client_maxfeerate} + m_client_maxfeerate{client_maxfeerate}, + m_allow_carveouts{allow_carveouts} { + // If we are using package feerates, we must be doing package submission. + // It also means carveouts and sibling eviction are not permitted. + if (m_package_feerates) { + Assume(m_package_submission); + Assume(!m_allow_carveouts); + Assume(!m_allow_sibling_eviction); + } + if (m_allow_sibling_eviction) Assume(m_allow_replacement); } }; @@ -576,8 +603,8 @@ public: /** * Submission of a subpackage. * If subpackage size == 1, calls AcceptSingleTransaction() with adjusted ATMPArgs to avoid - * package policy restrictions like no CPFP carve out (PackageMempoolChecks) and disabled RBF - * (m_allow_replacement), and creates a PackageMempoolAcceptResult wrapping the result. + * package policy restrictions like no CPFP carve out (PackageMempoolChecks) + * and creates a PackageMempoolAcceptResult wrapping the result. * * If subpackage size > 1, calls AcceptMultipleTransactions() with the provided ATMPArgs. * @@ -603,18 +630,11 @@ private: /** Iterators to mempool entries that this transaction directly conflicts with or may * replace via sibling eviction. */ CTxMemPool::setEntries m_iters_conflicting; - /** Iterators to all mempool entries that would be replaced by this transaction, including - * m_conflicts and their descendants. */ - CTxMemPool::setEntries m_all_conflicting; /** All mempool ancestors of this transaction. */ CTxMemPool::setEntries m_ancestors; /** Mempool entry constructed for this transaction. Constructed in PreChecks() but not * inserted into the mempool until Finalize(). */ std::unique_ptr<CTxMemPoolEntry> m_entry; - /** Pointers to the transactions that have been removed from the mempool and replaced by - * this transaction (everything in m_all_conflicting), used to return to the MemPoolAccept caller. Only populated if - * validation is successful and the original transactions are removed. */ - std::list<CTransactionRef> m_replaced_transactions; /** Whether RBF-related data structures (m_conflicts, m_iters_conflicting, m_all_conflicting, * m_replaced_transactions) include a sibling in addition to txns with conflicting inputs. */ bool m_sibling_eviction{false}; @@ -626,10 +646,6 @@ private: CAmount m_base_fees; /** Base fees + any fee delta set by the user with prioritisetransaction. */ CAmount m_modified_fees; - /** Total modified fees of all transactions being replaced. */ - CAmount m_conflicting_fees{0}; - /** Total virtual size of all transactions being replaced. */ - size_t m_conflicting_size{0}; /** If we're doing package validation (i.e. m_package_feerates=true), the "effective" * package feerate of this transaction is the total fees divided by the total size of @@ -651,12 +667,13 @@ private: // only tests that are fast should be done here (to avoid CPU DoS). bool PreChecks(ATMPArgs& args, Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); - // Run checks for mempool replace-by-fee. + // Run checks for mempool replace-by-fee, only used in AcceptSingleTransaction. bool ReplacementChecks(Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); // Enforce package mempool ancestor/descendant limits (distinct from individual - // ancestor/descendant limits done in PreChecks). + // ancestor/descendant limits done in PreChecks) and run Package RBF checks. bool PackageMempoolChecks(const std::vector<CTransactionRef>& txns, + std::vector<Workspace>& workspaces, int64_t total_vsize, PackageValidationState& package_state) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); @@ -707,9 +724,39 @@ private: Chainstate& m_active_chainstate; - /** Whether the transaction(s) would replace any mempool transactions and/or evict any siblings. - * If so, RBF rules apply. */ - bool m_rbf{false}; + // Fields below are per *sub*package state and must be reset prior to subsequent + // AcceptSingleTransaction and AcceptMultipleTransactions invocations + struct SubPackageState { + /** Aggregated modified fees of all transactions, used to calculate package feerate. */ + CAmount m_total_modified_fees{0}; + /** Aggregated virtual size of all transactions, used to calculate package feerate. */ + int64_t m_total_vsize{0}; + + // RBF-related members + /** Whether the transaction(s) would replace any mempool transactions and/or evict any siblings. + * If so, RBF rules apply. */ + bool m_rbf{false}; + /** All directly conflicting mempool transactions and their descendants. */ + CTxMemPool::setEntries m_all_conflicts; + /** Mempool transactions that were replaced. */ + std::list<CTransactionRef> m_replaced_transactions; + + /** Total modified fees of mempool transactions being replaced. */ + CAmount m_conflicting_fees{0}; + /** Total size (in virtual bytes) of mempool transactions being replaced. */ + size_t m_conflicting_size{0}; + }; + + struct SubPackageState m_subpackage; + + /** Re-set sub-package state to not leak between evaluations */ + void ClearSubPackageState() EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs) + { + m_subpackage = SubPackageState{}; + + // And clean coins while at it + CleanupTemporaryCoins(); + } }; bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) @@ -786,7 +833,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // // Replaceability signaling of the original transactions may be // ignored due to node setting. - const bool allow_rbf{m_pool.m_opts.full_rbf || SignalsOptInRBF(*ptxConflicting) || ptxConflicting->nVersion == 3}; + const bool allow_rbf{m_pool.m_opts.full_rbf || SignalsOptInRBF(*ptxConflicting) || ptxConflicting->version == TRUC_VERSION}; if (!allow_rbf) { return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "txn-mempool-conflict"); } @@ -890,7 +937,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // method of ensuring the tx remains bumped. For example, the fee-bumping child could disappear // due to a replacement. // The only exception is v3 transactions. - if (!bypass_limits && ws.m_ptx->nVersion != 3 && ws.m_modified_fees < m_pool.m_opts.min_relay_feerate.GetFee(ws.m_vsize)) { + if (!bypass_limits && ws.m_ptx->version != TRUC_VERSION && ws.m_modified_fees < m_pool.m_opts.min_relay_feerate.GetFee(ws.m_vsize)) { // Even though this is a fee-related failure, this result is TX_MEMPOOL_POLICY, not // TX_RECONSIDERABLE, because it cannot be bypassed using package validation. return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "min relay fee not met", @@ -904,11 +951,11 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) ws.m_iters_conflicting = m_pool.GetIterSet(ws.m_conflicts); // Note that these modifications are only applicable to single transaction scenarios; - // carve-outs and package RBF are disabled for multi-transaction evaluations. + // carve-outs are disabled for multi-transaction evaluations. CTxMemPool::Limits maybe_rbf_limits = m_pool.m_opts.limits; // Calculate in-mempool ancestors, up to a limit. - if (ws.m_conflicts.size() == 1) { + if (ws.m_conflicts.size() == 1 && args.m_allow_carveouts) { // In general, when we receive an RBF transaction with mempool conflicts, we want to know whether we // would meet the chain limits after the conflicts have been removed. However, there isn't a practical // way to do this short of calculating the ancestor and descendant sets with an overlay cache of @@ -947,11 +994,19 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) ws.m_ancestors = std::move(*ancestors); } else { // If CalculateMemPoolAncestors fails second time, we want the original error string. + const auto error_message{util::ErrorString(ancestors).original}; + + // Carve-out is not allowed in this context; fail + if (!args.m_allow_carveouts) { + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "too-long-mempool-chain", error_message); + } + // Contracting/payment channels CPFP carve-out: // If the new transaction is relatively small (up to 40k weight) // and has at most one ancestor (ie ancestor limit of 2, including // the new transaction), allow it if its parent has exactly the - // descendant limit descendants. + // descendant limit descendants. The transaction also cannot be v3, + // as its topology restrictions do not allow a second child. // // This allows protocols which rely on distrusting counterparties // being able to broadcast descendants of an unconfirmed transaction @@ -964,8 +1019,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) .descendant_count = maybe_rbf_limits.descendant_count + 1, .descendant_size_vbytes = maybe_rbf_limits.descendant_size_vbytes + EXTRA_DESCENDANT_TX_SIZE_LIMIT, }; - const auto error_message{util::ErrorString(ancestors).original}; - if (ws.m_vsize > EXTRA_DESCENDANT_TX_SIZE_LIMIT) { + if (ws.m_vsize > EXTRA_DESCENDANT_TX_SIZE_LIMIT || ws.m_ptx->version == TRUC_VERSION) { return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "too-long-mempool-chain", error_message); } if (auto ancestors_retry{m_pool.CalculateMemPoolAncestors(*entry, cpfp_carve_out_limits)}) { @@ -979,8 +1033,11 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // check using the full ancestor set here because it's more convenient to use what we have // already calculated. if (const auto err{SingleV3Checks(ws.m_ptx, ws.m_ancestors, ws.m_conflicts, ws.m_vsize)}) { - // Disabled within package validation. - if (err->second != nullptr && args.m_allow_replacement) { + // Single transaction contexts only. + if (args.m_allow_sibling_eviction && err->second != nullptr) { + // We should only be considering where replacement is considered valid as well. + Assume(args.m_allow_replacement); + // Potential sibling eviction. Add the sibling to our list of mempool conflicts to be // included in RBF checks. ws.m_conflicts.insert(err->second->GetHash()); @@ -1008,7 +1065,8 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-spends-conflicting-tx", *err_string); } - m_rbf = !ws.m_conflicts.empty(); + // We want to detect conflicts in any tx in a package to trigger package RBF logic + m_subpackage.m_rbf |= !ws.m_conflicts.empty(); return true; } @@ -1032,43 +1090,42 @@ bool MemPoolAccept::ReplacementChecks(Workspace& ws) // descendant transaction of a direct conflict to pay a higher feerate than the transaction that // might replace them, under these rules. if (const auto err_string{PaysMoreThanConflicts(ws.m_iters_conflicting, newFeeRate, hash)}) { - // Even though this is a fee-related failure, this result is TX_MEMPOOL_POLICY, not - // TX_RECONSIDERABLE, because it cannot be bypassed using package validation. - // This must be changed if package RBF is enabled. - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, + // This fee-related failure is TX_RECONSIDERABLE because validating in a package may change + // the result. + return state.Invalid(TxValidationResult::TX_RECONSIDERABLE, strprintf("insufficient fee%s", ws.m_sibling_eviction ? " (including sibling eviction)" : ""), *err_string); } // Calculate all conflicting entries and enforce Rule #5. - if (const auto err_string{GetEntriesForConflicts(tx, m_pool, ws.m_iters_conflicting, ws.m_all_conflicting)}) { + if (const auto err_string{GetEntriesForConflicts(tx, m_pool, ws.m_iters_conflicting, m_subpackage.m_all_conflicts)}) { return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, strprintf("too many potential replacements%s", ws.m_sibling_eviction ? " (including sibling eviction)" : ""), *err_string); } // Enforce Rule #2. - if (const auto err_string{HasNoNewUnconfirmed(tx, m_pool, ws.m_iters_conflicting)}) { + if (const auto err_string{HasNoNewUnconfirmed(tx, m_pool, m_subpackage.m_all_conflicts)}) { // Sibling eviction is only done for v3 transactions, which cannot have multiple ancestors. Assume(!ws.m_sibling_eviction); return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, strprintf("replacement-adds-unconfirmed%s", ws.m_sibling_eviction ? " (including sibling eviction)" : ""), *err_string); } + // Check if it's economically rational to mine this transaction rather than the ones it // replaces and pays for its own relay fees. Enforce Rules #3 and #4. - for (CTxMemPool::txiter it : ws.m_all_conflicting) { - ws.m_conflicting_fees += it->GetModifiedFee(); - ws.m_conflicting_size += it->GetTxSize(); + for (CTxMemPool::txiter it : m_subpackage.m_all_conflicts) { + m_subpackage.m_conflicting_fees += it->GetModifiedFee(); + m_subpackage.m_conflicting_size += it->GetTxSize(); } - if (const auto err_string{PaysForRBF(ws.m_conflicting_fees, ws.m_modified_fees, ws.m_vsize, + if (const auto err_string{PaysForRBF(m_subpackage.m_conflicting_fees, ws.m_modified_fees, ws.m_vsize, m_pool.m_opts.incremental_relay_feerate, hash)}) { - // Even though this is a fee-related failure, this result is TX_MEMPOOL_POLICY, not - // TX_RECONSIDERABLE, because it cannot be bypassed using package validation. - // This must be changed if package RBF is enabled. - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, + // Result may change in a package context + return state.Invalid(TxValidationResult::TX_RECONSIDERABLE, strprintf("insufficient fee%s", ws.m_sibling_eviction ? " (including sibling eviction)" : ""), *err_string); } return true; } bool MemPoolAccept::PackageMempoolChecks(const std::vector<CTransactionRef>& txns, + std::vector<Workspace>& workspaces, const int64_t total_vsize, PackageValidationState& package_state) { @@ -1079,12 +1136,88 @@ bool MemPoolAccept::PackageMempoolChecks(const std::vector<CTransactionRef>& txn assert(std::all_of(txns.cbegin(), txns.cend(), [this](const auto& tx) { return !m_pool.exists(GenTxid::Txid(tx->GetHash()));})); + assert(txns.size() == workspaces.size()); + auto result = m_pool.CheckPackageLimits(txns, total_vsize); if (!result) { // This is a package-wide error, separate from an individual transaction error. return package_state.Invalid(PackageValidationResult::PCKG_POLICY, "package-mempool-limits", util::ErrorString(result).original); } - return true; + + // No conflicts means we're finished. Further checks are all RBF-only. + if (!m_subpackage.m_rbf) return true; + + // We're in package RBF context; replacement proposal must be size 2 + if (workspaces.size() != 2 || !Assume(IsChildWithParents(txns))) { + return package_state.Invalid(PackageValidationResult::PCKG_POLICY, "package RBF failed: package must be 1-parent-1-child"); + } + + // If the package has in-mempool ancestors, we won't consider a package RBF + // since it would result in a cluster larger than 2. + // N.B. To relax this constraint we will need to revisit how CCoinsViewMemPool::PackageAddTransaction + // is being used inside AcceptMultipleTransactions to track available inputs while processing a package. + for (const auto& ws : workspaces) { + if (!ws.m_ancestors.empty()) { + return package_state.Invalid(PackageValidationResult::PCKG_POLICY, "package RBF failed: new transaction cannot have mempool ancestors"); + } + } + + // Aggregate all conflicts into one set. + CTxMemPool::setEntries direct_conflict_iters; + for (Workspace& ws : workspaces) { + // Aggregate all conflicts into one set. + direct_conflict_iters.merge(ws.m_iters_conflicting); + } + + const auto& parent_ws = workspaces[0]; + const auto& child_ws = workspaces[1]; + + // Don't consider replacements that would cause us to remove a large number of mempool entries. + // This limit is not increased in a package RBF. Use the aggregate number of transactions. + if (const auto err_string{GetEntriesForConflicts(*child_ws.m_ptx, m_pool, direct_conflict_iters, + m_subpackage.m_all_conflicts)}) { + return package_state.Invalid(PackageValidationResult::PCKG_POLICY, + "package RBF failed: too many potential replacements", *err_string); + } + + for (CTxMemPool::txiter it : m_subpackage.m_all_conflicts) { + m_subpackage.m_conflicting_fees += it->GetModifiedFee(); + m_subpackage.m_conflicting_size += it->GetTxSize(); + } + + // Use the child as the transaction for attributing errors to. + const Txid& child_hash = child_ws.m_ptx->GetHash(); + if (const auto err_string{PaysForRBF(/*original_fees=*/m_subpackage.m_conflicting_fees, + /*replacement_fees=*/m_subpackage.m_total_modified_fees, + /*replacement_vsize=*/m_subpackage.m_total_vsize, + m_pool.m_opts.incremental_relay_feerate, child_hash)}) { + return package_state.Invalid(PackageValidationResult::PCKG_POLICY, + "package RBF failed: insufficient anti-DoS fees", *err_string); + } + + // Ensure this two transaction package is a "chunk" on its own; we don't want the child + // to be only paying anti-DoS fees + const CFeeRate parent_feerate(parent_ws.m_modified_fees, parent_ws.m_vsize); + const CFeeRate package_feerate(m_subpackage.m_total_modified_fees, m_subpackage.m_total_vsize); + if (package_feerate <= parent_feerate) { + return package_state.Invalid(PackageValidationResult::PCKG_POLICY, + "package RBF failed: package feerate is less than parent feerate", + strprintf("package feerate %s <= parent feerate is %s", package_feerate.ToString(), parent_feerate.ToString())); + } + + // Check if it's economically rational to mine this package rather than the ones it replaces. + // This takes the place of ReplacementChecks()'s PaysMoreThanConflicts() in the package RBF setting. + if (const auto err_tup{ImprovesFeerateDiagram(m_pool, direct_conflict_iters, m_subpackage.m_all_conflicts, m_subpackage.m_total_modified_fees, m_subpackage.m_total_vsize)}) { + return package_state.Invalid(PackageValidationResult::PCKG_POLICY, + "package RBF failed: " + err_tup.value().second, ""); + } + + LogPrint(BCLog::TXPACKAGES, "package RBF checks passed: parent %s (wtxid=%s), child %s (wtxid=%s)\n", + txns.front()->GetHash().ToString(), txns.front()->GetWitnessHash().ToString(), + txns.back()->GetHash().ToString(), txns.back()->GetWitnessHash().ToString()); + + + return true; } bool MemPoolAccept::PolicyScriptChecks(const ATMPArgs& args, Workspace& ws) @@ -1156,19 +1289,21 @@ bool MemPoolAccept::Finalize(const ATMPArgs& args, Workspace& ws) const uint256& hash = ws.m_hash; TxValidationState& state = ws.m_state; const bool bypass_limits = args.m_bypass_limits; - std::unique_ptr<CTxMemPoolEntry>& entry = ws.m_entry; + if (!m_subpackage.m_all_conflicts.empty()) Assume(args.m_allow_replacement); // Remove conflicting transactions from the mempool - for (CTxMemPool::txiter it : ws.m_all_conflicting) + for (CTxMemPool::txiter it : m_subpackage.m_all_conflicts) { - LogPrint(BCLog::MEMPOOL, "replacing tx %s (wtxid=%s) with %s (wtxid=%s) for %s additional fees, %d delta bytes\n", + LogPrint(BCLog::MEMPOOL, "replacing mempool tx %s (wtxid=%s, fees=%s, vsize=%s). New tx %s (wtxid=%s, fees=%s, vsize=%s)\n", it->GetTx().GetHash().ToString(), it->GetTx().GetWitnessHash().ToString(), + it->GetFee(), + it->GetTxSize(), hash.ToString(), tx.GetWitnessHash().ToString(), - FormatMoney(ws.m_modified_fees - ws.m_conflicting_fees), - (int)entry->GetTxSize() - (int)ws.m_conflicting_size); + entry->GetFee(), + entry->GetTxSize()); TRACE7(mempool, replaced, it->GetTx().GetHash().data(), it->GetTxSize(), @@ -1178,9 +1313,12 @@ bool MemPoolAccept::Finalize(const ATMPArgs& args, Workspace& ws) entry->GetTxSize(), entry->GetFee() ); - ws.m_replaced_transactions.push_back(it->GetSharedTx()); + m_subpackage.m_replaced_transactions.push_back(it->GetSharedTx()); } - m_pool.RemoveStaged(ws.m_all_conflicting, false, MemPoolRemovalReason::REPLACED); + m_pool.RemoveStaged(m_subpackage.m_all_conflicts, false, MemPoolRemovalReason::REPLACED); + // Don't attempt to process the same conflicts repeatedly during subpackage evaluation: + // they no longer exist on subsequent calls to Finalize() post-RemoveStaged + m_subpackage.m_all_conflicts.clear(); // Store transaction in memory m_pool.addUnchecked(*entry, ws.m_ancestors); @@ -1259,6 +1397,13 @@ bool MemPoolAccept::SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& std::transform(workspaces.cbegin(), workspaces.cend(), std::back_inserter(all_package_wtxids), [](const auto& ws) { return ws.m_ptx->GetWitnessHash(); }); + if (!m_subpackage.m_replaced_transactions.empty()) { + LogPrint(BCLog::MEMPOOL, "replaced %u mempool transactions with %u new one(s) for %s additional fees, %d delta bytes\n", + m_subpackage.m_replaced_transactions.size(), workspaces.size(), + m_subpackage.m_total_modified_fees - m_subpackage.m_conflicting_fees, + m_subpackage.m_total_vsize - static_cast<int>(m_subpackage.m_conflicting_size)); + } + // Add successful results. The returned results may change later if LimitMempoolSize() evicts them. for (Workspace& ws : workspaces) { const auto effective_feerate = args.m_package_feerates ? ws.m_package_feerate : @@ -1266,7 +1411,7 @@ bool MemPoolAccept::SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& const auto effective_feerate_wtxids = args.m_package_feerates ? all_package_wtxids : std::vector<Wtxid>{ws.m_ptx->GetWitnessHash()}; results.emplace(ws.m_ptx->GetWitnessHash(), - MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, + MempoolAcceptResult::Success(std::move(m_subpackage.m_replaced_transactions), ws.m_vsize, ws.m_base_fees, effective_feerate, effective_feerate_wtxids)); if (!m_pool.m_opts.signals) continue; const CTransaction& tx = *ws.m_ptx; @@ -1302,7 +1447,13 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef return MempoolAcceptResult::Failure(ws.m_state); } - if (m_rbf && !ReplacementChecks(ws)) return MempoolAcceptResult::Failure(ws.m_state); + if (m_subpackage.m_rbf && !ReplacementChecks(ws)) { + if (ws.m_state.GetResult() == TxValidationResult::TX_RECONSIDERABLE) { + // Failed for incentives-based fee reasons. Provide the effective feerate and which tx was included. + return MempoolAcceptResult::FeeFailure(ws.m_state, CFeeRate(ws.m_modified_fees, ws.m_vsize), single_wtxid); + } + return MempoolAcceptResult::Failure(ws.m_state); + } // Perform the inexpensive checks first and avoid hashing and signature verification unless // those checks pass, to mitigate CPU exhaustion denial-of-service attacks. @@ -1313,7 +1464,7 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef const CFeeRate effective_feerate{ws.m_modified_fees, static_cast<uint32_t>(ws.m_vsize)}; // Tx was accepted, but not added if (args.m_test_accept) { - return MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, + return MempoolAcceptResult::Success(std::move(m_subpackage.m_replaced_transactions), ws.m_vsize, ws.m_base_fees, effective_feerate, single_wtxid); } @@ -1334,7 +1485,14 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef m_pool.m_opts.signals->TransactionAddedToMempool(tx_info, m_pool.GetAndIncrementSequence()); } - return MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, ws.m_base_fees, + if (!m_subpackage.m_replaced_transactions.empty()) { + LogPrint(BCLog::MEMPOOL, "replaced %u mempool transactions with 1 new transaction for %s additional fees, %d delta bytes\n", + m_subpackage.m_replaced_transactions.size(), + ws.m_modified_fees - m_subpackage.m_conflicting_fees, + ws.m_vsize - static_cast<int>(m_subpackage.m_conflicting_size)); + } + + return MempoolAcceptResult::Success(std::move(m_subpackage.m_replaced_transactions), ws.m_vsize, ws.m_base_fees, effective_feerate, single_wtxid); } @@ -1375,10 +1533,14 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: } // Make the coins created by this transaction available for subsequent transactions in the - // package to spend. Since we already checked conflicts in the package and we don't allow - // replacements, we don't need to track the coins spent. Note that this logic will need to be - // updated if package replace-by-fee is allowed in the future. - assert(!args.m_allow_replacement); + // package to spend. If there are no conflicts within the package, no transaction can spend a coin + // needed by another transaction in the package. We also need to make sure that no package + // tx replaces (or replaces the ancestor of) the parent of another package tx. As long as we + // check these two things, we don't need to track the coins spent. + // If a package tx conflicts with a mempool tx, PackageMempoolChecks() ensures later that any package RBF attempt + // has *no* in-mempool ancestors, so we don't have to worry about subsequent transactions in + // same package spending the same in-mempool outpoints. This needs to be revisited for general + // package RBF. m_viewmempool.PackageAddTransaction(ws.m_ptx); } @@ -1400,28 +1562,26 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: // a child that is below mempool minimum feerate. To avoid these behaviors, callers of // AcceptMultipleTransactions need to restrict txns topology (e.g. to ancestor sets) and check // the feerates of individuals and subsets. - const auto m_total_vsize = std::accumulate(workspaces.cbegin(), workspaces.cend(), int64_t{0}, + m_subpackage.m_total_vsize = std::accumulate(workspaces.cbegin(), workspaces.cend(), int64_t{0}, [](int64_t sum, auto& ws) { return sum + ws.m_vsize; }); - const auto m_total_modified_fees = std::accumulate(workspaces.cbegin(), workspaces.cend(), CAmount{0}, + m_subpackage.m_total_modified_fees = std::accumulate(workspaces.cbegin(), workspaces.cend(), CAmount{0}, [](CAmount sum, auto& ws) { return sum + ws.m_modified_fees; }); - const CFeeRate package_feerate(m_total_modified_fees, m_total_vsize); + const CFeeRate package_feerate(m_subpackage.m_total_modified_fees, m_subpackage.m_total_vsize); std::vector<Wtxid> all_package_wtxids; all_package_wtxids.reserve(workspaces.size()); std::transform(workspaces.cbegin(), workspaces.cend(), std::back_inserter(all_package_wtxids), [](const auto& ws) { return ws.m_ptx->GetWitnessHash(); }); TxValidationState placeholder_state; if (args.m_package_feerates && - !CheckFeeRate(m_total_vsize, m_total_modified_fees, placeholder_state)) { + !CheckFeeRate(m_subpackage.m_total_vsize, m_subpackage.m_total_modified_fees, placeholder_state)) { package_state.Invalid(PackageValidationResult::PCKG_TX, "transaction failed"); return PackageMempoolAcceptResult(package_state, {{workspaces.back().m_ptx->GetWitnessHash(), - MempoolAcceptResult::FeeFailure(placeholder_state, CFeeRate(m_total_modified_fees, m_total_vsize), all_package_wtxids)}}); + MempoolAcceptResult::FeeFailure(placeholder_state, CFeeRate(m_subpackage.m_total_modified_fees, m_subpackage.m_total_vsize), all_package_wtxids)}}); } // Apply package mempool ancestor/descendant limits. Skip if there is only one transaction, - // because it's unnecessary. Also, CPFP carve out can increase the limit for individual - // transactions, but this exemption is not extended to packages in CheckPackageLimits(). - std::string err_string; - if (txns.size() > 1 && !PackageMempoolChecks(txns, m_total_vsize, package_state)) { + // because it's unnecessary. + if (txns.size() > 1 && !PackageMempoolChecks(txns, workspaces, m_subpackage.m_total_vsize, package_state)) { return PackageMempoolAcceptResult(package_state, std::move(results)); } @@ -1439,7 +1599,7 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: const auto effective_feerate_wtxids = args.m_package_feerates ? all_package_wtxids : std::vector<Wtxid>{ws.m_ptx->GetWitnessHash()}; results.emplace(ws.m_ptx->GetWitnessHash(), - MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), + MempoolAcceptResult::Success(std::move(m_subpackage.m_replaced_transactions), ws.m_vsize, ws.m_base_fees, effective_feerate, effective_feerate_wtxids)); } @@ -1504,7 +1664,8 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptSubPackage(const std::vector<CTr // Clean up m_view and m_viewmempool so that other subpackage evaluations don't have access to // coins they shouldn't. Keep some coins in order to minimize re-fetching coins from the UTXO set. - CleanupTemporaryCoins(); + // Clean up package feerate and rbf calculations + ClearSubPackageState(); return result; } @@ -1688,7 +1849,6 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, MempoolAcceptResult AcceptToMemoryPool(Chainstate& active_chainstate, const CTransactionRef& tx, int64_t accept_time, bool bypass_limits, bool test_accept) - EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { AssertLockHeld(::cs_main); const CChainParams& chainparams{active_chainstate.m_chainman.GetParams()}; @@ -1862,9 +2022,11 @@ void Chainstate::CheckForkWarningConditions() if (m_chainman.m_best_invalid && m_chainman.m_best_invalid->nChainWork > m_chain.Tip()->nChainWork + (GetBlockProof(*m_chain.Tip()) * 6)) { LogPrintf("%s: Warning: Found invalid chain at least ~6 blocks longer than our best chain.\nChain state database corruption likely.\n", __func__); - SetfLargeWorkInvalidChainFound(true); + m_chainman.GetNotifications().warningSet( + kernel::Warning::LARGE_WORK_INVALID_CHAIN, + _("Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade.")); } else { - SetfLargeWorkInvalidChainFound(false); + m_chainman.GetNotifications().warningUnset(kernel::Warning::LARGE_WORK_INVALID_CHAIN); } } @@ -2641,7 +2803,7 @@ bool Chainstate::FlushStateToDisk( CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(); LOCK(m_blockman.cs_LastBlockFile); - if (m_blockman.IsPruneMode() && (m_blockman.m_check_for_pruning || nManualPruneHeight > 0) && !m_chainman.m_blockman.m_reindexing) { + if (m_blockman.IsPruneMode() && (m_blockman.m_check_for_pruning || nManualPruneHeight > 0) && m_chainman.m_blockman.m_blockfiles_indexed) { // make sure we don't prune above any of the prune locks bestblocks // pruning is height-based int last_prune{m_chain.Height()}; // last height we can prune @@ -2788,13 +2950,6 @@ void Chainstate::PruneAndFlush() } } -/** Private helper function that concatenates warning messages. */ -static void AppendWarning(bilingual_str& res, const bilingual_str& warn) -{ - if (!res.empty()) res += Untranslated(", "); - res += warn; -} - static void UpdateTipLog( const CCoinsViewCache& coins_tip, const CBlockIndex* tip, @@ -2845,7 +3000,7 @@ void Chainstate::UpdateTip(const CBlockIndex* pindexNew) g_best_block_cv.notify_all(); } - bilingual_str warning_messages; + std::vector<bilingual_str> warning_messages; if (!m_chainman.IsInitialBlockDownload()) { const CBlockIndex* pindex = pindexNew; for (int bit = 0; bit < VERSIONBITS_NUM_BITS; bit++) { @@ -2854,14 +3009,15 @@ void Chainstate::UpdateTip(const CBlockIndex* pindexNew) if (state == ThresholdState::ACTIVE || state == ThresholdState::LOCKED_IN) { const bilingual_str warning = strprintf(_("Unknown new rules activated (versionbit %i)"), bit); if (state == ThresholdState::ACTIVE) { - m_chainman.GetNotifications().warning(warning); + m_chainman.GetNotifications().warningSet(kernel::Warning::UNKNOWN_NEW_RULES_ACTIVATED, warning); } else { - AppendWarning(warning_messages, warning); + warning_messages.push_back(warning); } } } } - UpdateTipLog(coins_tip, pindexNew, params, __func__, "", warning_messages.original); + UpdateTipLog(coins_tip, pindexNew, params, __func__, "", + util::Join(warning_messages, Untranslated(", ")).original); } /** Disconnect m_chain's tip. @@ -3254,10 +3410,10 @@ bool Chainstate::ActivateBestChainStep(BlockValidationState& state, CBlockIndex* return true; } -static SynchronizationState GetSynchronizationState(bool init, bool reindexing) +static SynchronizationState GetSynchronizationState(bool init, bool blockfiles_indexed) { if (!init) return SynchronizationState::POST_INIT; - if (reindexing) return SynchronizationState::INIT_REINDEX; + if (!blockfiles_indexed) return SynchronizationState::INIT_REINDEX; return SynchronizationState::INIT_DOWNLOAD; } @@ -3279,7 +3435,7 @@ static bool NotifyHeaderTip(ChainstateManager& chainman) LOCKS_EXCLUDED(cs_main) } // Send block tip changed notifications without cs_main if (fNotify) { - chainman.GetNotifications().headerTip(GetSynchronizationState(fInitialBlockDownload, chainman.m_blockman.m_reindexing), pindexHeader->nHeight, pindexHeader->nTime, false); + chainman.GetNotifications().headerTip(GetSynchronizationState(fInitialBlockDownload, chainman.m_blockman.m_blockfiles_indexed), pindexHeader->nHeight, pindexHeader->nTime, false); } return fNotify; } @@ -3398,7 +3554,7 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< } // Always notify the UI if a new block tip was connected - if (kernel::IsInterrupted(m_chainman.GetNotifications().blockTip(GetSynchronizationState(still_in_ibd, m_chainman.m_blockman.m_reindexing), *pindexNewTip))) { + if (kernel::IsInterrupted(m_chainman.GetNotifications().blockTip(GetSynchronizationState(still_in_ibd, m_chainman.m_blockman.m_blockfiles_indexed), *pindexNewTip))) { // Just breaking and returning success for now. This could // be changed to bubble up the kernel::Interrupted value to // the caller so the caller could distinguish between @@ -3624,7 +3780,7 @@ bool Chainstate::InvalidateBlock(BlockValidationState& state, CBlockIndex* pinde // parameter indicating the source of the tip change so hooks can // distinguish user-initiated invalidateblock changes from other // changes. - (void)m_chainman.GetNotifications().blockTip(GetSynchronizationState(m_chainman.IsInitialBlockDownload(), m_chainman.m_blockman.m_reindexing), *to_mark_failed->pprev); + (void)m_chainman.GetNotifications().blockTip(GetSynchronizationState(m_chainman.IsInitialBlockDownload(), m_chainman.m_blockman.m_blockfiles_indexed), *to_mark_failed->pprev); } return true; } @@ -4263,7 +4419,7 @@ void ChainstateManager::ReportHeadersPresync(const arith_uint256& work, int64_t m_last_presync_update = now; } bool initial_download = IsInitialBlockDownload(); - GetNotifications().headerTip(GetSynchronizationState(initial_download, m_blockman.m_reindexing), height, timestamp, /*presync=*/true); + GetNotifications().headerTip(GetSynchronizationState(initial_download, m_blockman.m_blockfiles_indexed), height, timestamp, /*presync=*/true); if (initial_download) { int64_t blocks_left{(NodeClock::now() - NodeSeconds{std::chrono::seconds{timestamp}}) / GetConsensus().PowTargetSpacing()}; blocks_left = std::max<int64_t>(0, blocks_left); @@ -4790,8 +4946,7 @@ bool ChainstateManager::LoadBlockIndex() { AssertLockHeld(cs_main); // Load block index from databases - bool needs_init = m_blockman.m_reindexing; - if (!m_blockman.m_reindexing) { + if (m_blockman.m_blockfiles_indexed) { bool ret{m_blockman.LoadBlockIndexDB(SnapshotBlockhash())}; if (!ret) return false; @@ -4822,18 +4977,6 @@ bool ChainstateManager::LoadBlockIndex() if (pindex->IsValid(BLOCK_VALID_TREE) && (m_best_header == nullptr || CBlockIndexWorkComparator()(m_best_header, pindex))) m_best_header = pindex; } - - needs_init = m_blockman.m_block_index.empty(); - } - - if (needs_init) { - // Everything here is for *new* reindex/DBs. Thus, though - // LoadBlockIndexDB may have set m_reindexing if we shut down - // mid-reindex previously, we don't check m_reindexing and - // instead only check it prior to LoadBlockIndexDB to set - // needs_init. - - LogPrintf("Initializing databases...\n"); } return true; } @@ -4974,7 +5117,7 @@ void ChainstateManager::LoadExternalBlockFile( } } - if (m_blockman.IsPruneMode() && !m_blockman.m_reindexing && pblock) { + if (m_blockman.IsPruneMode() && m_blockman.m_blockfiles_indexed && pblock) { // must update the tip for pruning to work while importing with -loadblock. // this is a tradeoff to conserve disk space at the expense of time // spent updating the tip to be able to prune. @@ -5046,6 +5189,14 @@ void ChainstateManager::LoadExternalBlockFile( LogPrintf("Loaded %i blocks from external file in %dms\n", nLoaded, Ticks<std::chrono::milliseconds>(SteadyClock::now() - start)); } +bool ChainstateManager::ShouldCheckBlockIndex() const +{ + // Assert to verify Flatten() has been called. + if (!*Assert(m_options.check_block_index)) return false; + if (GetRand(*m_options.check_block_index) >= 1) return false; + return true; +} + void ChainstateManager::CheckBlockIndex() { if (!ShouldCheckBlockIndex()) { @@ -5062,19 +5213,28 @@ void ChainstateManager::CheckBlockIndex() return; } - // Build forward-pointing map of the entire block tree. + // Build forward-pointing data structure for the entire block tree. + // For performance reasons, indexes of the best header chain are stored in a vector (within CChain). + // All remaining blocks are stored in a multimap. + // The best header chain can differ from the active chain: E.g. its entries may belong to blocks that + // are not yet validated. + CChain best_hdr_chain; + assert(m_best_header); + best_hdr_chain.SetTip(*m_best_header); + std::multimap<CBlockIndex*,CBlockIndex*> forward; for (auto& [_, block_index] : m_blockman.m_block_index) { - forward.emplace(block_index.pprev, &block_index); + // Only save indexes in forward that are not part of the best header chain. + if (!best_hdr_chain.Contains(&block_index)) { + // Only genesis, which must be part of the best header chain, can have a nullptr parent. + assert(block_index.pprev); + forward.emplace(block_index.pprev, &block_index); + } } + assert(forward.size() + best_hdr_chain.Height() + 1 == m_blockman.m_block_index.size()); - assert(forward.size() == m_blockman.m_block_index.size()); - - std::pair<std::multimap<CBlockIndex*,CBlockIndex*>::iterator,std::multimap<CBlockIndex*,CBlockIndex*>::iterator> rangeGenesis = forward.equal_range(nullptr); - CBlockIndex *pindex = rangeGenesis.first->second; - rangeGenesis.first++; - assert(rangeGenesis.first == rangeGenesis.second); // There is only one index entry with parent nullptr. - + CBlockIndex* pindex = best_hdr_chain[0]; + assert(pindex); // Iterate over the entire block tree, using depth-first search. // Along the way, remember whether there are blocks on the path from genesis // block being explored which are the first to have certain properties. @@ -5286,14 +5446,21 @@ void ChainstateManager::CheckBlockIndex() // assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow // End: actual consistency checks. - // Try descending into the first subnode. + + // Try descending into the first subnode. Always process forks first and the best header chain after. snap_update_firsts(); std::pair<std::multimap<CBlockIndex*,CBlockIndex*>::iterator,std::multimap<CBlockIndex*,CBlockIndex*>::iterator> range = forward.equal_range(pindex); if (range.first != range.second) { - // A subnode was found. + // A subnode not part of the best header chain was found. pindex = range.first->second; nHeight++; continue; + } else if (best_hdr_chain.Contains(pindex)) { + // Descend further into best header chain. + nHeight++; + pindex = best_hdr_chain[nHeight]; + if (!pindex) break; // we are finished, since the best header chain is always processed last + continue; } // This is a leaf node. // Move upwards until we reach a node of which we have not yet visited the last child. @@ -5319,9 +5486,15 @@ void ChainstateManager::CheckBlockIndex() // Proceed to the next one. rangePar.first++; if (rangePar.first != rangePar.second) { - // Move to the sibling. + // Move to a sibling not part of the best header chain. pindex = rangePar.first->second; break; + } else if (pindexPar == best_hdr_chain[nHeight - 1]) { + // Move to pindex's sibling on the best-chain, if it has one. + pindex = best_hdr_chain[nHeight]; + // There will not be a next block if (and only if) parent block is the best header. + assert((pindex == nullptr) == (pindexPar == best_hdr_chain.Tip())); + break; } else { // Move up further. pindex = pindexPar; @@ -5331,8 +5504,8 @@ void ChainstateManager::CheckBlockIndex() } } - // Check that we actually traversed the entire map. - assert(nNodes == forward.size()); + // Check that we actually traversed the entire block index. + assert(nNodes == forward.size() + best_hdr_chain.Height() + 1); } std::string Chainstate::ToString() @@ -5665,69 +5838,81 @@ bool ChainstateManager::PopulateAndValidateSnapshot( return false; } - COutPoint outpoint; - Coin coin; const uint64_t coins_count = metadata.m_coins_count; uint64_t coins_left = metadata.m_coins_count; - LogPrintf("[snapshot] loading coins from snapshot %s\n", base_blockhash.ToString()); + LogPrintf("[snapshot] loading %d coins from snapshot %s\n", coins_left, base_blockhash.ToString()); int64_t coins_processed{0}; while (coins_left > 0) { try { - coins_file >> outpoint; - coins_file >> coin; - } catch (const std::ios_base::failure&) { - LogPrintf("[snapshot] bad snapshot format or truncated snapshot after deserializing %d coins\n", - coins_count - coins_left); - return false; - } - if (coin.nHeight > base_height || - outpoint.n >= std::numeric_limits<decltype(outpoint.n)>::max() // Avoid integer wrap-around in coinstats.cpp:ApplyHash - ) { - LogPrintf("[snapshot] bad snapshot data after deserializing %d coins\n", - coins_count - coins_left); - return false; - } - if (!MoneyRange(coin.out.nValue)) { - LogPrintf("[snapshot] bad snapshot data after deserializing %d coins - bad tx out value\n", - coins_count - coins_left); - return false; - } + Txid txid; + coins_file >> txid; + size_t coins_per_txid{0}; + coins_per_txid = ReadCompactSize(coins_file); - coins_cache.EmplaceCoinInternalDANGER(std::move(outpoint), std::move(coin)); + if (coins_per_txid > coins_left) { + LogPrintf("[snapshot] mismatch in coins count in snapshot metadata and actual snapshot data\n"); + return false; + } - --coins_left; - ++coins_processed; + for (size_t i = 0; i < coins_per_txid; i++) { + COutPoint outpoint; + Coin coin; + outpoint.n = static_cast<uint32_t>(ReadCompactSize(coins_file)); + outpoint.hash = txid; + coins_file >> coin; + if (coin.nHeight > base_height || + outpoint.n >= std::numeric_limits<decltype(outpoint.n)>::max() // Avoid integer wrap-around in coinstats.cpp:ApplyHash + ) { + LogPrintf("[snapshot] bad snapshot data after deserializing %d coins\n", + coins_count - coins_left); + return false; + } + if (!MoneyRange(coin.out.nValue)) { + LogPrintf("[snapshot] bad snapshot data after deserializing %d coins - bad tx out value\n", + coins_count - coins_left); + return false; + } + coins_cache.EmplaceCoinInternalDANGER(std::move(outpoint), std::move(coin)); - if (coins_processed % 1000000 == 0) { - LogPrintf("[snapshot] %d coins loaded (%.2f%%, %.2f MB)\n", - coins_processed, - static_cast<float>(coins_processed) * 100 / static_cast<float>(coins_count), - coins_cache.DynamicMemoryUsage() / (1000 * 1000)); - } + --coins_left; + ++coins_processed; - // Batch write and flush (if we need to) every so often. - // - // If our average Coin size is roughly 41 bytes, checking every 120,000 coins - // means <5MB of memory imprecision. - if (coins_processed % 120000 == 0) { - if (m_interrupt) { - return false; - } + if (coins_processed % 1000000 == 0) { + LogPrintf("[snapshot] %d coins loaded (%.2f%%, %.2f MB)\n", + coins_processed, + static_cast<float>(coins_processed) * 100 / static_cast<float>(coins_count), + coins_cache.DynamicMemoryUsage() / (1000 * 1000)); + } + + // Batch write and flush (if we need to) every so often. + // + // If our average Coin size is roughly 41 bytes, checking every 120,000 coins + // means <5MB of memory imprecision. + if (coins_processed % 120000 == 0) { + if (m_interrupt) { + return false; + } - const auto snapshot_cache_state = WITH_LOCK(::cs_main, - return snapshot_chainstate.GetCoinsCacheSizeState()); + const auto snapshot_cache_state = WITH_LOCK(::cs_main, + return snapshot_chainstate.GetCoinsCacheSizeState()); - if (snapshot_cache_state >= CoinsCacheSizeState::CRITICAL) { - // This is a hack - we don't know what the actual best block is, but that - // doesn't matter for the purposes of flushing the cache here. We'll set this - // to its correct value (`base_blockhash`) below after the coins are loaded. - coins_cache.SetBestBlock(GetRandHash()); + if (snapshot_cache_state >= CoinsCacheSizeState::CRITICAL) { + // This is a hack - we don't know what the actual best block is, but that + // doesn't matter for the purposes of flushing the cache here. We'll set this + // to its correct value (`base_blockhash`) below after the coins are loaded. + coins_cache.SetBestBlock(GetRandHash()); - // No need to acquire cs_main since this chainstate isn't being used yet. - FlushSnapshotToDisk(coins_cache, /*snapshot_loaded=*/false); + // No need to acquire cs_main since this chainstate isn't being used yet. + FlushSnapshotToDisk(coins_cache, /*snapshot_loaded=*/false); + } + } } + } catch (const std::ios_base::failure&) { + LogPrintf("[snapshot] bad snapshot format or truncated snapshot after deserializing %d coins\n", + coins_processed); + return false; } } @@ -5740,7 +5925,8 @@ bool ChainstateManager::PopulateAndValidateSnapshot( bool out_of_coins{false}; try { - coins_file >> outpoint; + std::byte left_over_byte; + coins_file >> left_over_byte; } catch (const std::ios_base::failure&) { // We expect an exception since we should be out of coins. out_of_coins = true; @@ -5878,8 +6064,8 @@ SnapshotCompletionResult ChainstateManager::MaybeCompleteSnapshotValidation() PACKAGE_NAME, snapshot_tip_height, snapshot_base_height, snapshot_base_height, PACKAGE_BUGREPORT ); - LogPrintf("[snapshot] !!! %s\n", user_error.original); - LogPrintf("[snapshot] deleting snapshot, reverting to validated chain, and stopping node\n"); + LogError("[snapshot] !!! %s\n", user_error.original); + LogError("[snapshot] deleting snapshot, reverting to validated chain, and stopping node\n"); m_active_chainstate = m_ibd_chainstate.get(); m_snapshot_chainstate->m_disabled = true; @@ -6231,7 +6417,7 @@ bool ChainstateManager::ValidatedSnapshotCleanup() fs::path p_old, fs::path p_new, const fs::filesystem_error& err) { - LogPrintf("Error renaming path (%s) -> (%s): %s\n", + LogError("[snapshot] Error renaming path (%s) -> (%s): %s\n", fs::PathToString(p_old), fs::PathToString(p_new), err.what()); GetNotifications().fatalError(strprintf(_( "Rename of '%s' -> '%s' failed. " diff --git a/src/validation.h b/src/validation.h index 28b045fe80..ab7891539a 100644 --- a/src/validation.h +++ b/src/validation.h @@ -885,6 +885,12 @@ private: CBlockIndex* m_best_invalid GUARDED_BY(::cs_main){nullptr}; //! Internal helper for ActivateSnapshot(). + //! + //! De-serialization of a snapshot that is created with + //! CreateUTXOSnapshot() in rpc/blockchain.cpp. + //! To reduce space the serialization format of the snapshot avoids + //! duplication of tx hashes. The code takes advantage of the guarantee by + //! leveldb that keys are lexicographically sorted. [[nodiscard]] bool PopulateAndValidateSnapshot( Chainstate& snapshot_chainstate, AutoFile& coins_file, @@ -932,7 +938,7 @@ public: const CChainParams& GetParams() const { return m_options.chainparams; } const Consensus::Params& GetConsensus() const { return m_options.chainparams.GetConsensus(); } - bool ShouldCheckBlockIndex() const { return *Assert(m_options.check_block_index); } + bool ShouldCheckBlockIndex() const; const arith_uint256& MinimumChainWork() const { return *Assert(m_options.minimum_chain_work); } const uint256& AssumedValidBlock() const { return *Assert(m_options.assumed_valid_block); } kernel::Notifications& GetNotifications() const { return m_options.notifications; }; diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 42615b5d42..f1706b6800 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -194,7 +194,7 @@ util::Result<SelectionResult> SelectCoinsBnB(std::vector<OutputGroup>& utxo_pool for (const size_t& i : best_selection) { result.AddInput(utxo_pool.at(i)); } - result.ComputeAndSetWaste(cost_of_change, cost_of_change, CAmount{0}); + result.RecalculateWaste(cost_of_change, cost_of_change, CAmount{0}); assert(best_waste == result.GetWaste()); return result; @@ -792,35 +792,6 @@ void OutputGroupTypeMap::Push(const OutputGroup& group, OutputType type, bool in } } -CAmount SelectionResult::GetSelectionWaste(CAmount change_cost, CAmount target, bool use_effective_value) -{ - // This function should not be called with empty inputs as that would mean the selection failed - assert(!m_selected_inputs.empty()); - - // Always consider the cost of spending an input now vs in the future. - CAmount waste = 0; - for (const auto& coin_ptr : m_selected_inputs) { - const COutput& coin = *coin_ptr; - waste += coin.GetFee() - coin.long_term_fee; - } - // Bump fee of whole selection may diverge from sum of individual bump fees - waste -= bump_fee_group_discount; - - if (change_cost) { - // Consider the cost of making change and spending it in the future - // If we aren't making change, the caller should've set change_cost to 0 - assert(change_cost > 0); - waste += change_cost; - } else { - // When we are not making change (change_cost == 0), consider the excess we are throwing away to fees - CAmount selected_effective_value = use_effective_value ? GetSelectedEffectiveValue() : GetSelectedValue(); - assert(selected_effective_value >= target); - waste += selected_effective_value - target; - } - - return waste; -} - CAmount GenerateChangeTarget(const CAmount payment_value, const CAmount change_fee, FastRandomContext& rng) { if (payment_value <= CHANGE_LOWER / 2) { @@ -839,16 +810,32 @@ void SelectionResult::SetBumpFeeDiscount(const CAmount discount) bump_fee_group_discount = discount; } - -void SelectionResult::ComputeAndSetWaste(const CAmount min_viable_change, const CAmount change_cost, const CAmount change_fee) +void SelectionResult::RecalculateWaste(const CAmount min_viable_change, const CAmount change_cost, const CAmount change_fee) { - const CAmount change = GetChange(min_viable_change, change_fee); + // This function should not be called with empty inputs as that would mean the selection failed + assert(!m_selected_inputs.empty()); + + // Always consider the cost of spending an input now vs in the future. + CAmount waste = 0; + for (const auto& coin_ptr : m_selected_inputs) { + const COutput& coin = *coin_ptr; + waste += coin.GetFee() - coin.long_term_fee; + } + // Bump fee of whole selection may diverge from sum of individual bump fees + waste -= bump_fee_group_discount; - if (change > 0) { - m_waste = GetSelectionWaste(change_cost, m_target, m_use_effective); + if (GetChange(min_viable_change, change_fee)) { + // if we have a minimum viable amount after deducting fees, account for + // cost of creating and spending change + waste += change_cost; } else { - m_waste = GetSelectionWaste(0, m_target, m_use_effective); + // When we are not making change (GetChange(…) == 0), consider the excess we are throwing away to fees + CAmount selected_effective_value = m_use_effective ? GetSelectedEffectiveValue() : GetSelectedValue(); + assert(selected_effective_value >= m_target); + waste += selected_effective_value - m_target; } + + m_waste = waste; } void SelectionResult::SetAlgoCompleted(bool algo_completed) diff --git a/src/wallet/coinselection.h b/src/wallet/coinselection.h index 80c92e1b0e..9fb000422c 100644 --- a/src/wallet/coinselection.h +++ b/src/wallet/coinselection.h @@ -350,22 +350,6 @@ private: } } - /** Compute the waste for this result given the cost of change - * and the opportunity cost of spending these inputs now vs in the future. - * If change exists, waste = change_cost + inputs * (effective_feerate - long_term_feerate) - * If no change, waste = excess + inputs * (effective_feerate - long_term_feerate) - * where excess = selected_effective_value - target - * change_cost = effective_feerate * change_output_size + long_term_feerate * change_spend_size - * - * @param[in] change_cost The cost of creating change and spending it in the future. - * Only used if there is change, in which case it must be positive. - * Must be 0 if there is no change. - * @param[in] target The amount targeted by the coin selection algorithm. - * @param[in] use_effective_value Whether to use the input's effective value (when true) or the real value (when false). - * @return The waste - */ - [[nodiscard]] CAmount GetSelectionWaste(CAmount change_cost, CAmount target, bool use_effective_value = true); - public: explicit SelectionResult(const CAmount target, SelectionAlgorithm algo) : m_target(target), m_algo(algo) {} @@ -387,8 +371,19 @@ public: /** How much individual inputs overestimated the bump fees for shared ancestries */ void SetBumpFeeDiscount(const CAmount discount); - /** Calculates and stores the waste for this selection via GetSelectionWaste */ - void ComputeAndSetWaste(const CAmount min_viable_change, const CAmount change_cost, const CAmount change_fee); + /** Calculates and stores the waste for this result given the cost of change + * and the opportunity cost of spending these inputs now vs in the future. + * If change exists, waste = change_cost + inputs * (effective_feerate - long_term_feerate) - bump_fee_group_discount + * If no change, waste = excess + inputs * (effective_feerate - long_term_feerate) - bump_fee_group_discount + * where excess = selected_effective_value - target + * change_cost = effective_feerate * change_output_size + long_term_feerate * change_spend_size + * + * @param[in] min_viable_change The minimum amount necessary to make a change output economic + * @param[in] change_cost The cost of creating a change output and spending it in the future. Only + * used if there is change, in which case it must be non-negative. + * @param[in] change_fee The fee for creating a change output + */ + void RecalculateWaste(const CAmount min_viable_change, const CAmount change_cost, const CAmount change_fee); [[nodiscard]] CAmount GetWaste() const; /** Tracks that algorithm was able to exhaustively search the entire combination space before hitting limit of tries */ diff --git a/src/wallet/external_signer_scriptpubkeyman.cpp b/src/wallet/external_signer_scriptpubkeyman.cpp index b5703fa54a..32e9941453 100644 --- a/src/wallet/external_signer_scriptpubkeyman.cpp +++ b/src/wallet/external_signer_scriptpubkeyman.cpp @@ -6,6 +6,7 @@ #include <common/args.h> #include <common/system.h> #include <external_signer.h> +#include <node/types.h> #include <wallet/external_signer_scriptpubkeyman.h> #include <iostream> @@ -17,6 +18,8 @@ #include <utility> #include <vector> +using common::PSBTError; + namespace wallet { bool ExternalSignerScriptPubKeyMan::SetupDescriptor(WalletBatch& batch, std::unique_ptr<Descriptor> desc) { @@ -76,7 +79,7 @@ util::Result<void> ExternalSignerScriptPubKeyMan::DisplayAddress(const CTxDestin } // If sign is true, transaction must previously have been filled -TransactionError ExternalSignerScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type, bool sign, bool bip32derivs, int* n_signed, bool finalize) const +std::optional<PSBTError> ExternalSignerScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type, bool sign, bool bip32derivs, int* n_signed, bool finalize) const { if (!sign) { return DescriptorScriptPubKeyMan::FillPSBT(psbt, txdata, sighash_type, false, bip32derivs, n_signed, finalize); @@ -88,14 +91,14 @@ TransactionError ExternalSignerScriptPubKeyMan::FillPSBT(PartiallySignedTransact // TODO: for multisig wallets, we should only care if all _our_ inputs are signed complete &= PSBTInputSigned(input); } - if (complete) return TransactionError::OK; + if (complete) return {}; std::string strFailReason; if(!GetExternalSigner().SignTransaction(psbt, strFailReason)) { tfm::format(std::cerr, "Failed to sign: %s\n", strFailReason); - return TransactionError::EXTERNAL_SIGNER_FAILED; + return PSBTError::EXTERNAL_SIGNER_FAILED; } if (finalize) FinalizePSBT(psbt); // This won't work in a multisig setup - return TransactionError::OK; + return {}; } } // namespace wallet diff --git a/src/wallet/external_signer_scriptpubkeyman.h b/src/wallet/external_signer_scriptpubkeyman.h index 44286456b6..10d67d2ab4 100644 --- a/src/wallet/external_signer_scriptpubkeyman.h +++ b/src/wallet/external_signer_scriptpubkeyman.h @@ -35,7 +35,7 @@ class ExternalSignerScriptPubKeyMan : public DescriptorScriptPubKeyMan */ util::Result<void> DisplayAddress(const CTxDestination& dest, const ExternalSigner& signer) const; - TransactionError FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const override; + std::optional<common::PSBTError> FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const override; }; } // namespace wallet #endif // BITCOIN_WALLET_EXTERNAL_SIGNER_SCRIPTPUBKEYMAN_H diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 6a8453965b..3184d0f3b0 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -5,6 +5,7 @@ #include <common/system.h> #include <consensus/validation.h> #include <interfaces/chain.h> +#include <node/types.h> #include <policy/fees.h> #include <policy/policy.h> #include <util/moneystr.h> @@ -92,7 +93,7 @@ static feebumper::Result CheckFeeRate(const CWallet& wallet, const CMutableTrans } CAmount new_total_fee = newFeerate.GetFee(maxTxSize) + combined_bump_fee.value(); - CFeeRate incrementalRelayFee = std::max(wallet.chain().relayIncrementalFee(), CFeeRate(WALLET_INCREMENTAL_RELAY_FEE)); + CFeeRate incrementalRelayFee = wallet.chain().relayIncrementalFee(); // Min total fee is old fee + relay fee CAmount minTotalFee = old_fee + incrementalRelayFee.GetFee(maxTxSize); @@ -343,8 +344,8 @@ bool SignTransaction(CWallet& wallet, CMutableTransaction& mtx) { // so external signers are not asked to sign more than once. bool complete; wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, false /* sign */, true /* bip32derivs */); - const TransactionError err = wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, true /* sign */, false /* bip32derivs */); - if (err != TransactionError::OK) return false; + auto err{wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, true /* sign */, false /* bip32derivs */)}; + if (err) return false; complete = FinalizeAndExtractPSBT(psbtx, mtx); return complete; } else { diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index 0c1cae7253..9fab1b2ee4 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -8,6 +8,7 @@ #include <consensus/amount.h> #include <interfaces/chain.h> #include <interfaces/handler.h> +#include <node/types.h> #include <policy/fees.h> #include <primitives/transaction.h> #include <rpc/server.h> @@ -34,6 +35,7 @@ #include <utility> #include <vector> +using common::PSBTError; using interfaces::Chain; using interfaces::FoundBlock; using interfaces::Handler; @@ -389,7 +391,7 @@ public: } return {}; } - TransactionError fillPSBT(int sighash_type, + std::optional<PSBTError> fillPSBT(int sighash_type, bool sign, bool bip32derivs, size_t* n_signed, diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 8b78a670e4..fe35f6b223 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -21,6 +21,8 @@ #include <system_error> +using util::Join; + namespace wallet { bool VerifyWallets(WalletContext& context) { diff --git a/src/wallet/migrate.cpp b/src/wallet/migrate.cpp index 09254a76ad..d7d8577374 100644 --- a/src/wallet/migrate.cpp +++ b/src/wallet/migrate.cpp @@ -551,7 +551,7 @@ void BerkeleyRODatabase::Open() // } // Check the last page number - uint32_t expected_last_page = (size / page_size) - 1; + uint32_t expected_last_page{uint32_t((size / page_size) - 1)}; if (outer_meta.last_page != expected_last_page) { throw std::runtime_error("Last page number could not fit in file"); } diff --git a/src/wallet/rpc/addresses.cpp b/src/wallet/rpc/addresses.cpp index 65587f0b18..0c2ad06eea 100644 --- a/src/wallet/rpc/addresses.cpp +++ b/src/wallet/rpc/addresses.cpp @@ -206,9 +206,9 @@ RPCHelpMan listaddressgroupings() addressInfo.push_back(address_book_entry->GetLabel()); } } - jsonGrouping.push_back(addressInfo); + jsonGrouping.push_back(std::move(addressInfo)); } - jsonGroupings.push_back(jsonGrouping); + jsonGroupings.push_back(std::move(jsonGrouping)); } return jsonGroupings; }, @@ -287,9 +287,30 @@ RPCHelpMan addmultisigaddress() output_type = parsed.value(); } - // Construct using pay-to-script-hash: + // Construct multisig scripts + FlatSigningProvider provider; CScript inner; - CTxDestination dest = AddAndGetMultisigDestination(required, pubkeys, output_type, spk_man, inner); + CTxDestination dest = AddAndGetMultisigDestination(required, pubkeys, output_type, provider, inner); + + // Import scripts into the wallet + for (const auto& [id, script] : provider.scripts) { + // Due to a bug in the legacy wallet, the p2sh maximum script size limit is also imposed on 'p2sh-segwit' and 'bech32' redeem scripts. + // Even when redeem scripts over MAX_SCRIPT_ELEMENT_SIZE bytes are valid for segwit output types, we don't want to + // enable it because: + // 1) It introduces a compatibility-breaking change requiring downgrade protection; older wallets would be unable to interact with these "new" legacy wallets. + // 2) Considering the ongoing deprecation of the legacy spkm, this issue adds another good reason to transition towards descriptors. + if (script.size() > MAX_SCRIPT_ELEMENT_SIZE) throw JSONRPCError(RPC_WALLET_ERROR, "Unsupported multisig script size for legacy wallet. Upgrade to descriptors to overcome this limitation for p2sh-segwit or bech32 scripts"); + + if (!spk_man.AddCScript(script)) { + if (CScript inner_script; spk_man.GetCScript(CScriptID(script), inner_script)) { + CHECK_NONFATAL(inner_script == script); // Nothing to add, script already contained by the wallet + continue; + } + throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Error importing script into the wallet")); + } + } + + // Store destination in the addressbook pwallet->SetAddressBook(dest, label, AddressPurpose::SEND); // Make the descriptor @@ -407,9 +428,9 @@ public: // Only when the script corresponds to an address. UniValue subobj(UniValue::VOBJ); UniValue detail = DescribeAddress(embedded); - subobj.pushKVs(detail); + subobj.pushKVs(std::move(detail)); UniValue wallet_detail = std::visit(*this, embedded); - subobj.pushKVs(wallet_detail); + subobj.pushKVs(std::move(wallet_detail)); subobj.pushKV("address", EncodeDestination(embedded)); subobj.pushKV("scriptPubKey", HexStr(subscript)); // Always report the pubkey at the top level, so that `getnewaddress()['pubkey']` always works. @@ -490,7 +511,7 @@ static UniValue DescribeWalletAddress(const CWallet& wallet, const CTxDestinatio CScript script = GetScriptForDestination(dest); std::unique_ptr<SigningProvider> provider = nullptr; provider = wallet.GetSolvingProvider(script); - ret.pushKVs(detail); + ret.pushKVs(std::move(detail)); ret.pushKVs(std::visit(DescribeWalletAddressVisitor(provider.get()), dest)); return ret; } @@ -607,7 +628,7 @@ RPCHelpMan getaddressinfo() ret.pushKV("iswatchonly", bool(mine & ISMINE_WATCH_ONLY)); UniValue detail = DescribeWalletAddress(*pwallet, dest); - ret.pushKVs(detail); + ret.pushKVs(std::move(detail)); ret.pushKV("ischange", ScriptIsChange(*pwallet, scriptPubKey)); @@ -688,7 +709,7 @@ RPCHelpMan getaddressesbylabel() // which currently is O(1). UniValue value(UniValue::VOBJ); value.pushKV("purpose", _purpose ? PurposeToString(*_purpose) : "unknown"); - ret.pushKVEnd(address, value); + ret.pushKVEnd(address, std::move(value)); } }); diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 8d3eea59ee..8cddb8b099 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -34,6 +34,7 @@ using interfaces::FoundBlock; +using util::SplitString; namespace wallet { std::string static EncodeDumpString(const std::string &str) { @@ -1827,16 +1828,16 @@ RPCHelpMan listdescriptors() UniValue range(UniValue::VARR); range.push_back(info.range->first); range.push_back(info.range->second - 1); - spk.pushKV("range", range); + spk.pushKV("range", std::move(range)); spk.pushKV("next", info.next_index); spk.pushKV("next_index", info.next_index); } - descriptors.push_back(spk); + descriptors.push_back(std::move(spk)); } UniValue response(UniValue::VOBJ); response.pushKV("wallet_name", wallet->GetName()); - response.pushKV("descriptors", descriptors); + response.pushKV("descriptors", std::move(descriptors)); return response; }, diff --git a/src/wallet/rpc/coins.cpp b/src/wallet/rpc/coins.cpp index b6c7396f4b..2cf94a5722 100644 --- a/src/wallet/rpc/coins.cpp +++ b/src/wallet/rpc/coins.cpp @@ -416,7 +416,7 @@ RPCHelpMan listlockunspent() o.pushKV("txid", outpt.hash.GetHex()); o.pushKV("vout", (int)outpt.n); - ret.push_back(o); + ret.push_back(std::move(o)); } return ret; @@ -477,7 +477,7 @@ RPCHelpMan getbalances() const auto full_bal = GetBalance(wallet, 0, false); balances_mine.pushKV("used", ValueFromAmount(full_bal.m_mine_trusted + full_bal.m_mine_untrusted_pending - bal.m_mine_trusted - bal.m_mine_untrusted_pending)); } - balances.pushKV("mine", balances_mine); + balances.pushKV("mine", std::move(balances_mine)); } auto spk_man = wallet.GetLegacyScriptPubKeyMan(); if (spk_man && spk_man->HaveWatchOnly()) { @@ -485,7 +485,7 @@ RPCHelpMan getbalances() balances_watchonly.pushKV("trusted", ValueFromAmount(bal.m_watchonly_trusted)); balances_watchonly.pushKV("untrusted_pending", ValueFromAmount(bal.m_watchonly_untrusted_pending)); balances_watchonly.pushKV("immature", ValueFromAmount(bal.m_watchonly_immature)); - balances.pushKV("watchonly", balances_watchonly); + balances.pushKV("watchonly", std::move(balances_watchonly)); } AppendLastProcessedBlock(balances, wallet); @@ -724,7 +724,7 @@ RPCHelpMan listunspent() PushParentDescriptors(*pwallet, scriptPubKey, entry); if (avoid_reuse) entry.pushKV("reused", reused); entry.pushKV("safe", out.safe); - results.push_back(entry); + results.push_back(std::move(entry)); } return results; diff --git a/src/wallet/rpc/signmessage.cpp b/src/wallet/rpc/signmessage.cpp index c9fb693482..edf93ecab7 100644 --- a/src/wallet/rpc/signmessage.cpp +++ b/src/wallet/rpc/signmessage.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/signmessage.h> #include <key_io.h> #include <rpc/util.h> -#include <util/message.h> #include <wallet/rpc/util.h> #include <wallet/wallet.h> diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index 1a364a75ed..ac2a4826f0 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -2,14 +2,15 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/messages.h> #include <consensus/validation.h> #include <core_io.h> #include <key_io.h> +#include <node/types.h> #include <policy/policy.h> #include <rpc/rawtransaction_util.h> #include <rpc/util.h> #include <script/script.h> -#include <util/fees.h> #include <util/rbf.h> #include <util/translation.h> #include <util/vector.h> @@ -22,6 +23,12 @@ #include <univalue.h> +using common::FeeModeFromString; +using common::FeeModes; +using common::InvalidEstimateModeErrorMessage; +using common::StringForFeeReason; +using common::TransactionErrorString; +using node::TransactionError; namespace wallet { std::vector<CRecipient> CreateRecipients(const std::vector<std::pair<CTxDestination, CAmount>>& outputs, const std::set<int>& subtract_fee_outputs) @@ -97,9 +104,9 @@ static UniValue FinishTransaction(const std::shared_ptr<CWallet> pwallet, const // so external signers are not asked to sign more than once. bool complete; pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, /*sign=*/false, /*bip32derivs=*/true); - const TransactionError err{pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, /*sign=*/true, /*bip32derivs=*/false)}; - if (err != TransactionError::OK) { - throw JSONRPCTransactionError(err); + const auto err{pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, /*sign=*/true, /*bip32derivs=*/false)}; + if (err) { + throw JSONRPCPSBTError(*err); } CMutableTransaction mtx; @@ -718,7 +725,7 @@ static void SetOptionsInputWeights(const UniValue& inputs, UniValue& options) weights.push_back(input); } } - options.pushKV("input_weights", weights); + options.pushKV("input_weights", std::move(weights)); } RPCHelpMan fundrawtransaction() @@ -1153,8 +1160,8 @@ static RPCHelpMan bumpfee_helper(std::string method_name) } else { PartiallySignedTransaction psbtx(mtx); bool complete = false; - const TransactionError err = pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, /*sign=*/false, /*bip32derivs=*/true); - CHECK_NONFATAL(err == TransactionError::OK); + const auto err{pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, /*sign=*/false, /*bip32derivs=*/true)}; + CHECK_NONFATAL(!err); CHECK_NONFATAL(!complete); DataStream ssTx{}; ssTx << psbtx; @@ -1167,7 +1174,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) for (const bilingual_str& error : errors) { result_errors.push_back(error.original); } - result.pushKV("errors", result_errors); + result.pushKV("errors", std::move(result_errors)); return result; }, @@ -1295,7 +1302,7 @@ RPCHelpMan sendall() { return RPCHelpMan{"sendall", "EXPERIMENTAL warning: this call may be changed in future releases.\n" - "\nSpend the value of all (or specific) confirmed UTXOs in the wallet to one or more recipients.\n" + "\nSpend the value of all (or specific) confirmed UTXOs and unconfirmed change in the wallet to one or more recipients.\n" "Unconfirmed inbound UTXOs and locked UTXOs will not be spent. Sendall will respect the avoid_reuse wallet flag.\n" "If your wallet contains many small inputs, either because it received tiny payments or as a result of accumulating change, consider using `send_max` to exclude inputs that are worth less than the fees needed to spend them.\n", { @@ -1388,7 +1395,7 @@ RPCHelpMan sendall() if (recipient.isStr()) { UniValue rkvp(UniValue::VOBJ); rkvp.pushKV(recipient.get_str(), 0); - recipient_key_value_pairs.push_back(rkvp); + recipient_key_value_pairs.push_back(std::move(rkvp)); addresses_without_amount.insert(recipient.get_str()); } else { recipient_key_value_pairs.push_back(recipient); @@ -1470,10 +1477,18 @@ RPCHelpMan sendall() } } + std::vector<COutPoint> outpoints_spent; + outpoints_spent.reserve(rawTx.vin.size()); + + for (const CTxIn& tx_in : rawTx.vin) { + outpoints_spent.push_back(tx_in.prevout); + } + // estimate final size of tx const TxSize tx_size{CalculateMaximumSignedTxSize(CTransaction(rawTx), pwallet.get())}; const CAmount fee_from_size{fee_rate.GetFee(tx_size.vsize)}; - const CAmount effective_value{total_input_value - fee_from_size}; + const std::optional<CAmount> total_bump_fees{pwallet->chain().calculateCombinedBumpFee(outpoints_spent, fee_rate)}; + CAmount effective_value = total_input_value - fee_from_size - total_bump_fees.value_or(0); if (fee_from_size > pwallet->m_default_max_tx_fee) { throw JSONRPCError(RPC_WALLET_ERROR, TransactionErrorString(TransactionError::MAX_FEE_EXCEEDED).original); @@ -1602,9 +1617,9 @@ RPCHelpMan walletprocesspsbt() if (sign) EnsureWalletIsUnlocked(*pwallet); - const TransactionError err{wallet.FillPSBT(psbtx, complete, nHashType, sign, bip32derivs, nullptr, finalize)}; - if (err != TransactionError::OK) { - throw JSONRPCTransactionError(err); + const auto err{wallet.FillPSBT(psbtx, complete, nHashType, sign, bip32derivs, nullptr, finalize)}; + if (err) { + throw JSONRPCPSBTError(*err); } UniValue result(UniValue::VOBJ); @@ -1736,9 +1751,9 @@ RPCHelpMan walletcreatefundedpsbt() // Fill transaction with out data but don't sign bool bip32derivs = request.params[4].isNull() ? true : request.params[4].get_bool(); bool complete = true; - const TransactionError err{wallet.FillPSBT(psbtx, complete, 1, /*sign=*/false, /*bip32derivs=*/bip32derivs)}; - if (err != TransactionError::OK) { - throw JSONRPCTransactionError(err); + const auto err{wallet.FillPSBT(psbtx, complete, 1, /*sign=*/false, /*bip32derivs=*/bip32derivs)}; + if (err) { + throw JSONRPCPSBTError(*err); } // Serialize the PSBT diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index 05b340995d..5abc983701 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -39,11 +39,11 @@ static void WalletTxToJSON(const CWallet& wallet, const CWalletTx& wtx, UniValue UniValue conflicts(UniValue::VARR); for (const uint256& conflict : wallet.GetTxConflicts(wtx)) conflicts.push_back(conflict.GetHex()); - entry.pushKV("walletconflicts", conflicts); + entry.pushKV("walletconflicts", std::move(conflicts)); UniValue mempool_conflicts(UniValue::VARR); for (const Txid& mempool_conflict : wtx.mempool_conflicts) mempool_conflicts.push_back(mempool_conflict.GetHex()); - entry.pushKV("mempoolconflicts", mempool_conflicts); + entry.pushKV("mempoolconflicts", std::move(mempool_conflicts)); entry.pushKV("time", wtx.GetTxTime()); entry.pushKV("timereceived", int64_t{wtx.nTimeReceived}); @@ -172,8 +172,8 @@ static UniValue ListReceived(const CWallet& wallet, const UniValue& params, cons transactions.push_back(_item.GetHex()); } } - obj.pushKV("txids", transactions); - ret.push_back(obj); + obj.pushKV("txids", std::move(transactions)); + ret.push_back(std::move(obj)); } }; @@ -195,7 +195,7 @@ static UniValue ListReceived(const CWallet& wallet, const UniValue& params, cons obj.pushKV("amount", ValueFromAmount(nAmount)); obj.pushKV("confirmations", (nConf == std::numeric_limits<int>::max() ? 0 : nConf)); obj.pushKV("label", entry.first); - ret.push_back(obj); + ret.push_back(std::move(obj)); } } @@ -353,7 +353,7 @@ static void ListTransactions(const CWallet& wallet, const CWalletTx& wtx, int nM if (fLong) WalletTxToJSON(wallet, wtx, entry); entry.pushKV("abandoned", wtx.isAbandoned()); - ret.push_back(entry); + ret.push_back(std::move(entry)); } } @@ -396,7 +396,7 @@ static void ListTransactions(const CWallet& wallet, const CWalletTx& wtx, int nM entry.pushKV("abandoned", wtx.isAbandoned()); if (fLong) WalletTxToJSON(wallet, wtx, entry); - ret.push_back(entry); + ret.push_back(std::move(entry)); } } } @@ -682,8 +682,8 @@ RPCHelpMan listsinceblock() CHECK_NONFATAL(wallet.chain().findAncestorByHeight(wallet.GetLastBlockHash(), wallet.GetLastBlockHeight() + 1 - target_confirms, FoundBlock().hash(lastblock))); UniValue ret(UniValue::VOBJ); - ret.pushKV("transactions", transactions); - if (include_removed) ret.pushKV("removed", removed); + ret.pushKV("transactions", std::move(transactions)); + if (include_removed) ret.pushKV("removed", std::move(removed)); ret.pushKV("lastblock", lastblock.GetHex()); return ret; @@ -789,14 +789,14 @@ RPCHelpMan gettransaction() UniValue details(UniValue::VARR); ListTransactions(*pwallet, wtx, 0, false, details, filter, /*filter_label=*/std::nullopt); - entry.pushKV("details", details); + entry.pushKV("details", std::move(details)); entry.pushKV("hex", EncodeHexTx(*wtx.tx)); if (verbose) { UniValue decoded(UniValue::VOBJ); TxToUniv(*wtx.tx, /*block_hash=*/uint256(), /*entry=*/decoded, /*include_hex=*/false); - entry.pushKV("decoded", decoded); + entry.pushKV("decoded", std::move(decoded)); } AppendLastProcessedBlock(entry, *pwallet); diff --git a/src/wallet/rpc/util.cpp b/src/wallet/rpc/util.cpp index 1252843e9d..67b5ae0fe2 100644 --- a/src/wallet/rpc/util.cpp +++ b/src/wallet/rpc/util.cpp @@ -149,7 +149,7 @@ void PushParentDescriptors(const CWallet& wallet, const CScript& script_pubkey, for (const auto& desc: wallet.GetWalletDescriptors(script_pubkey)) { parent_descs.push_back(desc.descriptor->ToString()); } - entry.pushKV("parent_descs", parent_descs); + entry.pushKV("parent_descs", std::move(parent_descs)); } void HandleWalletError(const std::shared_ptr<CWallet> wallet, DatabaseStatus& status, bilingual_str& error) @@ -179,13 +179,13 @@ void HandleWalletError(const std::shared_ptr<CWallet> wallet, DatabaseStatus& st } } -void AppendLastProcessedBlock(UniValue& entry, const CWallet& wallet) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +void AppendLastProcessedBlock(UniValue& entry, const CWallet& wallet) { AssertLockHeld(wallet.cs_wallet); UniValue lastprocessedblock{UniValue::VOBJ}; lastprocessedblock.pushKV("hash", wallet.GetLastBlockHash().GetHex()); lastprocessedblock.pushKV("height", wallet.GetLastBlockHeight()); - entry.pushKV("lastprocessedblock", lastprocessedblock); + entry.pushKV("lastprocessedblock", std::move(lastprocessedblock)); } } // namespace wallet diff --git a/src/wallet/rpc/wallet.cpp b/src/wallet/rpc/wallet.cpp index f1cb595271..8c218ad766 100644 --- a/src/wallet/rpc/wallet.cpp +++ b/src/wallet/rpc/wallet.cpp @@ -128,7 +128,7 @@ static RPCHelpMan getwalletinfo() UniValue scanning(UniValue::VOBJ); scanning.pushKV("duration", Ticks<std::chrono::seconds>(pwallet->ScanningDuration())); scanning.pushKV("progress", pwallet->ScanningProgress()); - obj.pushKV("scanning", scanning); + obj.pushKV("scanning", std::move(scanning)); } else { obj.pushKV("scanning", false); } @@ -172,11 +172,11 @@ static RPCHelpMan listwalletdir() for (const auto& path : ListDatabases(GetWalletDir())) { UniValue wallet(UniValue::VOBJ); wallet.pushKV("name", path.utf8string()); - wallets.push_back(wallet); + wallets.push_back(std::move(wallet)); } UniValue result(UniValue::VOBJ); - result.pushKV("wallets", wallets); + result.pushKV("wallets", std::move(wallets)); return result; }, }; @@ -395,7 +395,7 @@ static RPCHelpMan createwallet() if (!request.params[4].isNull() && request.params[4].get_bool()) { flags |= WALLET_FLAG_AVOID_REUSE; } - if (self.Arg<bool>(5)) { + if (self.Arg<bool>("descriptors")) { #ifndef USE_SQLITE throw JSONRPCError(RPC_WALLET_ERROR, "Compiled without sqlite support (required for descriptor wallets)"); #endif @@ -489,7 +489,7 @@ static RPCHelpMan unloadwallet() // Release the "main" shared pointer and prevent further notifications. // Note that any attempt to load the same wallet would fail until the wallet // is destroyed (see CheckUniqueFileid). - std::optional<bool> load_on_start{self.MaybeArg<bool>(1)}; + std::optional<bool> load_on_start{self.MaybeArg<bool>("load_on_startup")}; if (!RemoveWallet(context, wallet, load_on_start, warnings)) { throw JSONRPCError(RPC_MISC_ERROR, "Requested wallet already unloaded"); } diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index b42275fe4b..c64aff5fe2 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -5,6 +5,7 @@ #include <hash.h> #include <key_io.h> #include <logging.h> +#include <node/types.h> #include <outputtype.h> #include <script/descriptor.h> #include <script/script.h> @@ -20,6 +21,9 @@ #include <optional> +using common::PSBTError; +using util::ToString; + namespace wallet { //! Value for the first BIP 32 hardened derivation. Can be used as a bit mask and as a value. See BIP 32 for more details. const uint32_t BIP32_HARDENED_KEY_LIMIT = 0x80000000; @@ -627,7 +631,7 @@ SigningResult LegacyScriptPubKeyMan::SignMessage(const std::string& message, con return SigningResult::SIGNING_FAILED; } -TransactionError LegacyScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbtx, const PrecomputedTransactionData& txdata, int sighash_type, bool sign, bool bip32derivs, int* n_signed, bool finalize) const +std::optional<PSBTError> LegacyScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbtx, const PrecomputedTransactionData& txdata, int sighash_type, bool sign, bool bip32derivs, int* n_signed, bool finalize) const { if (n_signed) { *n_signed = 0; @@ -642,13 +646,13 @@ TransactionError LegacyScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psb // Get the Sighash type if (sign && input.sighash_type != std::nullopt && *input.sighash_type != sighash_type) { - return TransactionError::SIGHASH_MISMATCH; + return PSBTError::SIGHASH_MISMATCH; } // Check non_witness_utxo has specified prevout if (input.non_witness_utxo) { if (txin.prevout.n >= input.non_witness_utxo->vout.size()) { - return TransactionError::MISSING_INPUTS; + return PSBTError::MISSING_INPUTS; } } else if (input.witness_utxo.IsNull()) { // There's no UTXO so we can just skip this now @@ -670,7 +674,7 @@ TransactionError LegacyScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psb UpdatePSBTOutput(HidingSigningProvider(this, true, !bip32derivs), psbtx, i); } - return TransactionError::OK; + return {}; } std::unique_ptr<CKeyMetadata> LegacyScriptPubKeyMan::GetMetadata(const CTxDestination& dest) const @@ -2485,7 +2489,7 @@ SigningResult DescriptorScriptPubKeyMan::SignMessage(const std::string& message, return SigningResult::OK; } -TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbtx, const PrecomputedTransactionData& txdata, int sighash_type, bool sign, bool bip32derivs, int* n_signed, bool finalize) const +std::optional<PSBTError> DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbtx, const PrecomputedTransactionData& txdata, int sighash_type, bool sign, bool bip32derivs, int* n_signed, bool finalize) const { if (n_signed) { *n_signed = 0; @@ -2500,7 +2504,7 @@ TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& // Get the Sighash type if (sign && input.sighash_type != std::nullopt && *input.sighash_type != sighash_type) { - return TransactionError::SIGHASH_MISMATCH; + return PSBTError::SIGHASH_MISMATCH; } // Get the scriptPubKey to know which SigningProvider to use @@ -2509,7 +2513,7 @@ TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& script = input.witness_utxo.scriptPubKey; } else if (input.non_witness_utxo) { if (txin.prevout.n >= input.non_witness_utxo->vout.size()) { - return TransactionError::MISSING_INPUTS; + return PSBTError::MISSING_INPUTS; } script = input.non_witness_utxo->vout[txin.prevout.n].scriptPubKey; } else { @@ -2580,7 +2584,7 @@ TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& UpdatePSBTOutput(HidingSigningProvider(keys.get(), /*hide_secret=*/true, /*hide_origin=*/!bip32derivs), psbtx, i); } - return TransactionError::OK; + return {}; } std::unique_ptr<CKeyMetadata> DescriptorScriptPubKeyMan::GetMetadata(const CTxDestination& dest) const diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 2c1ab8d44a..4d9f7bb1fa 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -6,13 +6,15 @@ #define BITCOIN_WALLET_SCRIPTPUBKEYMAN_H #include <addresstype.h> +#include <common/messages.h> +#include <common/signmessage.h> +#include <common/types.h> #include <logging.h> +#include <node/types.h> #include <psbt.h> #include <script/descriptor.h> #include <script/script.h> #include <script/signingprovider.h> -#include <util/error.h> -#include <util/message.h> #include <util/result.h> #include <util/time.h> #include <wallet/crypter.h> @@ -243,7 +245,7 @@ public: /** Sign a message with the given script */ virtual SigningResult SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const { return SigningResult::SIGNING_FAILED; }; /** Adds script and derivation path information to a PSBT, and optionally signs it. */ - virtual TransactionError FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = SIGHASH_DEFAULT, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const { return TransactionError::INVALID_PSBT; } + virtual std::optional<common::PSBTError> FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = SIGHASH_DEFAULT, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const { return common::PSBTError::UNSUPPORTED; } virtual uint256 GetID() const { return uint256(); } @@ -421,7 +423,7 @@ public: bool SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, bilingual_str>& input_errors) const override; SigningResult SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const override; - TransactionError FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = SIGHASH_DEFAULT, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const override; + std::optional<common::PSBTError> FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = SIGHASH_DEFAULT, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const override; uint256 GetID() const override; @@ -651,7 +653,7 @@ public: bool SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, bilingual_str>& input_errors) const override; SigningResult SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const override; - TransactionError FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = SIGHASH_DEFAULT, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const override; + std::optional<common::PSBTError> FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = SIGHASH_DEFAULT, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const override; uint256 GetID() const override; diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 9a7e166e68..b9b4666208 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -4,10 +4,12 @@ #include <algorithm> #include <common/args.h> +#include <common/messages.h> #include <common/system.h> #include <consensus/amount.h> #include <consensus/validation.h> #include <interfaces/chain.h> +#include <node/types.h> #include <numeric> #include <policy/policy.h> #include <primitives/transaction.h> @@ -15,7 +17,6 @@ #include <script/signingprovider.h> #include <script/solver.h> #include <util/check.h> -#include <util/fees.h> #include <util/moneystr.h> #include <util/rbf.h> #include <util/trace.h> @@ -29,7 +30,10 @@ #include <cmath> +using common::StringForFeeReason; +using common::TransactionErrorString; using interfaces::FoundBlock; +using node::TransactionError; namespace wallet { static constexpr size_t OUTPUT_GROUP_MAX_ENTRIES{100}; @@ -132,7 +136,7 @@ static std::optional<int64_t> GetSignedTxinWeight(const CWallet* wallet, const C // txouts needs to be in the order of tx.vin TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, const std::vector<CTxOut>& txouts, const CCoinControl* coin_control) { - // nVersion + nLockTime + input count + output count + // version + nLockTime + input count + output count int64_t weight = (4 + 4 + GetSizeOfCompactSize(tx.vin.size()) + GetSizeOfCompactSize(tx.vout.size())) * WITNESS_SCALE_FACTOR; // Whether any input spends a witness program. Necessary to run before the next loop over the // inputs in order to accurately compute the compactSize length for the witness data per input. @@ -256,7 +260,7 @@ static OutputType GetOutputType(TxoutType type, bool is_from_p2sh) // Fetch and validate the coin control selected inputs. // Coins could be internal (from the wallet) or external. util::Result<PreSelectedInputs> FetchSelectedInputs(const CWallet& wallet, const CCoinControl& coin_control, - const CoinSelectionParams& coin_selection_params) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) + const CoinSelectionParams& coin_selection_params) { PreSelectedInputs result; const bool can_grind_r = wallet.CanGrindR(); @@ -711,7 +715,7 @@ util::Result<SelectionResult> ChooseSelectionResult(interfaces::Chain& chain, co if (coin_selection_params.m_effective_feerate > CFeeRate{3 * coin_selection_params.m_long_term_feerate}) { // Minimize input set for feerates of at least 3×LTFRE (default: 30 ṩ/vB+) if (auto cg_result{CoinGrinder(groups.positive_group, nTargetValue, coin_selection_params.m_min_change_target, max_inputs_weight)}) { - cg_result->ComputeAndSetWaste(coin_selection_params.min_viable_change, coin_selection_params.m_cost_of_change, coin_selection_params.m_change_fee); + cg_result->RecalculateWaste(coin_selection_params.min_viable_change, coin_selection_params.m_cost_of_change, coin_selection_params.m_change_fee); results.push_back(*cg_result); } else { append_error(std::move(cg_result)); @@ -746,7 +750,7 @@ util::Result<SelectionResult> ChooseSelectionResult(interfaces::Chain& chain, co if (bump_fee_overestimate) { result.SetBumpFeeDiscount(bump_fee_overestimate); } - result.ComputeAndSetWaste(coin_selection_params.min_viable_change, coin_selection_params.m_cost_of_change, coin_selection_params.m_change_fee); + result.RecalculateWaste(coin_selection_params.min_viable_change, coin_selection_params.m_cost_of_change, coin_selection_params.m_change_fee); } // Choose the result with the least waste @@ -771,7 +775,7 @@ util::Result<SelectionResult> SelectCoins(const CWallet& wallet, CoinsResult& av if (selection_target <= 0) { SelectionResult result(nTargetValue, SelectionAlgorithm::MANUAL); result.AddInputs(pre_set_inputs.coins, coin_selection_params.m_subtract_fee_outputs); - result.ComputeAndSetWaste(coin_selection_params.min_viable_change, coin_selection_params.m_cost_of_change, coin_selection_params.m_change_fee); + result.RecalculateWaste(coin_selection_params.min_viable_change, coin_selection_params.m_cost_of_change, coin_selection_params.m_change_fee); return result; } @@ -792,7 +796,7 @@ util::Result<SelectionResult> SelectCoins(const CWallet& wallet, CoinsResult& av SelectionResult preselected(pre_set_inputs.total_amount, SelectionAlgorithm::MANUAL); preselected.AddInputs(pre_set_inputs.coins, coin_selection_params.m_subtract_fee_outputs); op_selection_result->Merge(preselected); - op_selection_result->ComputeAndSetWaste(coin_selection_params.min_viable_change, + op_selection_result->RecalculateWaste(coin_selection_params.min_viable_change, coin_selection_params.m_cost_of_change, coin_selection_params.m_change_fee); } @@ -985,7 +989,7 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( CMutableTransaction txNew; // The resulting transaction that we make if (coin_control.m_version) { - txNew.nVersion = coin_control.m_version.value(); + txNew.version = coin_control.m_version.value(); } CoinSelectionParams coin_selection_params{rng_fast}; // Parameters for coin selection, init with dummy @@ -1084,7 +1088,7 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( const auto change_spend_fee = coin_selection_params.m_discard_feerate.GetFee(coin_selection_params.change_spend_size); coin_selection_params.min_viable_change = std::max(change_spend_fee + 1, dust); - // Static vsize overhead + outputs vsize. 4 nVersion, 4 nLocktime, 1 input count, 1 witness overhead (dummy, flag, stack size) + // Static vsize overhead + outputs vsize. 4 version, 4 nLocktime, 1 input count, 1 witness overhead (dummy, flag, stack size) coin_selection_params.tx_noinputs_size = 10 + GetSizeOfCompactSize(vecSend.size()); // bytes for output count // vouts to the payees @@ -1394,7 +1398,7 @@ util::Result<CreatedTransactionResult> FundTransaction(CWallet& wallet, const CM coinControl.m_locktime = tx.nLockTime; // Set the user desired version - coinControl.m_version = tx.nVersion; + coinControl.m_version = tx.version; // Acquire the locks to prevent races to the new locked unspents between the // CreateTransaction call and LockCoin calls (when lockUnspents is true). diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 9a349f0992..7bd92b471c 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -874,29 +874,32 @@ BOOST_AUTO_TEST_CASE(SelectCoins_test) BOOST_AUTO_TEST_CASE(waste_test) { const CAmount fee{100}; + const CAmount min_viable_change{300}; const CAmount change_cost{125}; + const CAmount change_fee{30}; const CAmount fee_diff{40}; const CAmount in_amt{3 * COIN}; const CAmount target{2 * COIN}; - const CAmount excess{in_amt - fee * 2 - target}; + const CAmount excess{80}; + const CAmount exact_target{in_amt - fee * 2}; // Maximum spendable amount after fees: no change, no excess - // The following tests that the waste is calculated correctly in various scenarios. - // ComputeAndSetWaste will first determine the size of the change output. We don't really - // care about the change and just want to use the variant that always includes the change_cost, - // so min_viable_change and change_fee are set to 0 to ensure that. + // In the following, we test that the waste is calculated correctly in various scenarios. + // Usually, RecalculateWaste would compute change_fee and change_cost on basis of the + // change output type, current feerate, and discard_feerate, but we use fixed values + // across this test to make the test easier to understand. { // Waste with change is the change cost and difference between fee and long term fee SelectionResult selection1{target, SelectionAlgorithm::MANUAL}; - add_coin(1 * COIN, 1, selection1, fee, fee - fee_diff); + add_coin(1 * COIN, 1, selection1, /*fee=*/fee, /*long_term_fee=*/fee - fee_diff); add_coin(2 * COIN, 2, selection1, fee, fee - fee_diff); - selection1.ComputeAndSetWaste(/*min_viable_change=*/0, change_cost, /*change_fee=*/0); + selection1.RecalculateWaste(min_viable_change, change_cost, change_fee); BOOST_CHECK_EQUAL(fee_diff * 2 + change_cost, selection1.GetWaste()); // Waste will be greater when fee is greater, but long term fee is the same SelectionResult selection2{target, SelectionAlgorithm::MANUAL}; add_coin(1 * COIN, 1, selection2, fee * 2, fee - fee_diff); add_coin(2 * COIN, 2, selection2, fee * 2, fee - fee_diff); - selection2.ComputeAndSetWaste(/*min_viable_change=*/0, change_cost, /*change_fee=*/0); + selection2.RecalculateWaste(min_viable_change, change_cost, change_fee); BOOST_CHECK_GT(selection2.GetWaste(), selection1.GetWaste()); // Waste with change is the change cost and difference between fee and long term fee @@ -904,25 +907,25 @@ BOOST_AUTO_TEST_CASE(waste_test) SelectionResult selection3{target, SelectionAlgorithm::MANUAL}; add_coin(1 * COIN, 1, selection3, fee, fee + fee_diff); add_coin(2 * COIN, 2, selection3, fee, fee + fee_diff); - selection3.ComputeAndSetWaste(/*min_viable_change=*/0, change_cost, /*change_fee=*/0); + selection3.RecalculateWaste(min_viable_change, change_cost, change_fee); BOOST_CHECK_EQUAL(fee_diff * -2 + change_cost, selection3.GetWaste()); BOOST_CHECK_LT(selection3.GetWaste(), selection1.GetWaste()); } { // Waste without change is the excess and difference between fee and long term fee - SelectionResult selection_nochange1{target, SelectionAlgorithm::MANUAL}; + SelectionResult selection_nochange1{exact_target - excess, SelectionAlgorithm::MANUAL}; add_coin(1 * COIN, 1, selection_nochange1, fee, fee - fee_diff); add_coin(2 * COIN, 2, selection_nochange1, fee, fee - fee_diff); - selection_nochange1.ComputeAndSetWaste(/*min_viable_change=*/0, /*change_cost=*/0, /*change_fee=*/0); + selection_nochange1.RecalculateWaste(min_viable_change, change_cost, change_fee); BOOST_CHECK_EQUAL(fee_diff * 2 + excess, selection_nochange1.GetWaste()); // Waste without change is the excess and difference between fee and long term fee // With long term fee greater than fee, waste should be less than when long term fee is less than fee - SelectionResult selection_nochange2{target, SelectionAlgorithm::MANUAL}; + SelectionResult selection_nochange2{exact_target - excess, SelectionAlgorithm::MANUAL}; add_coin(1 * COIN, 1, selection_nochange2, fee, fee + fee_diff); add_coin(2 * COIN, 2, selection_nochange2, fee, fee + fee_diff); - selection_nochange2.ComputeAndSetWaste(/*min_viable_change=*/0, /*change_cost=*/0, /*change_fee=*/0); + selection_nochange2.RecalculateWaste(min_viable_change, change_cost, change_fee); BOOST_CHECK_EQUAL(fee_diff * -2 + excess, selection_nochange2.GetWaste()); BOOST_CHECK_LT(selection_nochange2.GetWaste(), selection_nochange1.GetWaste()); } @@ -932,57 +935,54 @@ BOOST_AUTO_TEST_CASE(waste_test) SelectionResult selection{target, SelectionAlgorithm::MANUAL}; add_coin(1 * COIN, 1, selection, fee, fee); add_coin(2 * COIN, 2, selection, fee, fee); - selection.ComputeAndSetWaste(/*min_viable_change=*/0, change_cost, /*change_fee=*/0); + selection.RecalculateWaste(min_viable_change, change_cost, change_fee); BOOST_CHECK_EQUAL(change_cost, selection.GetWaste()); } { // Waste without change and fee == long term fee is just the excess - SelectionResult selection{target, SelectionAlgorithm::MANUAL}; + SelectionResult selection{exact_target - excess, SelectionAlgorithm::MANUAL}; add_coin(1 * COIN, 1, selection, fee, fee); add_coin(2 * COIN, 2, selection, fee, fee); - selection.ComputeAndSetWaste(/*min_viable_change=*/0, /*change_cost=*/0, /*change_fee=*/0); + selection.RecalculateWaste(min_viable_change, change_cost, change_fee); BOOST_CHECK_EQUAL(excess, selection.GetWaste()); } { - // No Waste when fee == long_term_fee, no change, and no excess - const CAmount exact_target{in_amt - fee * 2}; + // Waste is 0 when fee == long_term_fee, no change, and no excess SelectionResult selection{exact_target, SelectionAlgorithm::MANUAL}; add_coin(1 * COIN, 1, selection, fee, fee); add_coin(2 * COIN, 2, selection, fee, fee); - selection.ComputeAndSetWaste(/*min_viable_change=*/0, /*change_cost=*/0, /*change_fee=*/0); + selection.RecalculateWaste(min_viable_change, change_cost , change_fee); BOOST_CHECK_EQUAL(0, selection.GetWaste()); } { - // No Waste when (fee - long_term_fee) == (-cost_of_change), and no excess + // Waste is 0 when (fee - long_term_fee) == (-cost_of_change), and no excess SelectionResult selection{target, SelectionAlgorithm::MANUAL}; - const CAmount new_change_cost{fee_diff * 2}; add_coin(1 * COIN, 1, selection, fee, fee + fee_diff); add_coin(2 * COIN, 2, selection, fee, fee + fee_diff); - selection.ComputeAndSetWaste(/*min_viable_change=*/0, new_change_cost, /*change_fee=*/0); + selection.RecalculateWaste(min_viable_change, /*change_cost=*/fee_diff * 2, change_fee); BOOST_CHECK_EQUAL(0, selection.GetWaste()); } { - // No Waste when (fee - long_term_fee) == (-excess), no change cost - const CAmount new_target{in_amt - fee * 2 - fee_diff * 2}; + // Waste is 0 when (fee - long_term_fee) == (-excess), no change cost + const CAmount new_target{exact_target - /*excess=*/fee_diff * 2}; SelectionResult selection{new_target, SelectionAlgorithm::MANUAL}; add_coin(1 * COIN, 1, selection, fee, fee + fee_diff); add_coin(2 * COIN, 2, selection, fee, fee + fee_diff); - selection.ComputeAndSetWaste(/*min_viable_change=*/0, /*change_cost=*/0, /*change_fee=*/0); + selection.RecalculateWaste(min_viable_change, change_cost, change_fee); BOOST_CHECK_EQUAL(0, selection.GetWaste()); } { // Negative waste when the long term fee is greater than the current fee and the selected value == target - const CAmount exact_target{3 * COIN - 2 * fee}; SelectionResult selection{exact_target, SelectionAlgorithm::MANUAL}; const CAmount target_waste1{-2 * fee_diff}; // = (2 * fee) - (2 * (fee + fee_diff)) add_coin(1 * COIN, 1, selection, fee, fee + fee_diff); add_coin(2 * COIN, 2, selection, fee, fee + fee_diff); - selection.ComputeAndSetWaste(/*min_viable_change=*/0, /*change_cost=*/0, /*change_fee=*/0); + selection.RecalculateWaste(min_viable_change, change_cost, change_fee); BOOST_CHECK_EQUAL(target_waste1, selection.GetWaste()); } @@ -990,10 +990,14 @@ BOOST_AUTO_TEST_CASE(waste_test) // Negative waste when the long term fee is greater than the current fee and change_cost < - (inputs * (fee - long_term_fee)) SelectionResult selection{target, SelectionAlgorithm::MANUAL}; const CAmount large_fee_diff{90}; - const CAmount target_waste2{-2 * large_fee_diff + change_cost}; // = (2 * fee) - (2 * (fee + large_fee_diff)) + change_cost + const CAmount target_waste2{-2 * large_fee_diff + change_cost}; + // = (2 * fee) - (2 * (fee + large_fee_diff)) + change_cost + // = (2 * 100) - (2 * (100 + 90)) + 125 + // = 200 - 380 + 125 = -55 + assert(target_waste2 == -55); add_coin(1 * COIN, 1, selection, fee, fee + large_fee_diff); add_coin(2 * COIN, 2, selection, fee, fee + large_fee_diff); - selection.ComputeAndSetWaste(/*min_viable_change=*/0, change_cost, /*change_fee=*/0); + selection.RecalculateWaste(min_viable_change, change_cost, change_fee); BOOST_CHECK_EQUAL(target_waste2, selection.GetWaste()); } } @@ -1018,12 +1022,12 @@ BOOST_AUTO_TEST_CASE(bump_fee_test) inputs[i]->ApplyBumpFee(20*(i+1)); } - selection.ComputeAndSetWaste(min_viable_change, change_cost, change_fee); + selection.RecalculateWaste(min_viable_change, change_cost, change_fee); CAmount expected_waste = fee_diff * -2 + change_cost + /*bump_fees=*/60; BOOST_CHECK_EQUAL(expected_waste, selection.GetWaste()); selection.SetBumpFeeDiscount(30); - selection.ComputeAndSetWaste(min_viable_change, change_cost, change_fee); + selection.RecalculateWaste(min_viable_change, change_cost, change_fee); expected_waste = fee_diff * -2 + change_cost + /*bump_fees=*/60 - /*group_discount=*/30; BOOST_CHECK_EQUAL(expected_waste, selection.GetWaste()); } @@ -1044,12 +1048,12 @@ BOOST_AUTO_TEST_CASE(bump_fee_test) inputs[i]->ApplyBumpFee(20*(i+1)); } - selection.ComputeAndSetWaste(min_viable_change, change_cost, change_fee); + selection.RecalculateWaste(min_viable_change, change_cost, change_fee); CAmount expected_waste = fee_diff * -2 + /*bump_fees=*/60 + /*excess = 100 - bump_fees*/40; BOOST_CHECK_EQUAL(expected_waste, selection.GetWaste()); selection.SetBumpFeeDiscount(30); - selection.ComputeAndSetWaste(min_viable_change, change_cost, change_fee); + selection.RecalculateWaste(min_viable_change, change_cost, change_fee); expected_waste = fee_diff * -2 + /*bump_fees=*/60 - /*group_discount=*/30 + /*excess = 100 - bump_fees + group_discount*/70; BOOST_CHECK_EQUAL(expected_waste, selection.GetWaste()); } @@ -1429,6 +1433,7 @@ BOOST_AUTO_TEST_CASE(check_max_weight) /*avoid_partial=*/false, }; + int max_weight = MAX_STANDARD_TX_WEIGHT - WITNESS_SCALE_FACTOR * (cs_params.tx_noinputs_size + cs_params.change_output_size); { // Scenario 1: // The actor starts with 1x 50.0 BTC and 1515x 0.033 BTC (~100.0 BTC total) unspent outputs @@ -1450,10 +1455,9 @@ BOOST_AUTO_TEST_CASE(check_max_weight) m_node); BOOST_CHECK(result); - // Verify that only the 50 BTC UTXO was selected - const auto& selection_res = result->GetInputSet(); - BOOST_CHECK(selection_res.size() == 1); - BOOST_CHECK((*selection_res.begin())->GetEffectiveValue() == 50 * COIN); + // Verify that the 50 BTC UTXO was selected, and result is below max_weight + BOOST_CHECK(has_coin(result->GetInputSet(), CAmount(50 * COIN))); + BOOST_CHECK_LE(result->GetWeight(), max_weight); } { @@ -1479,6 +1483,7 @@ BOOST_AUTO_TEST_CASE(check_max_weight) BOOST_CHECK(has_coin(result->GetInputSet(), CAmount(0.0625 * COIN))); BOOST_CHECK(has_coin(result->GetInputSet(), CAmount(0.025 * COIN))); + BOOST_CHECK_LE(result->GetWeight(), max_weight); } { diff --git a/src/wallet/test/fuzz/coinselection.cpp b/src/wallet/test/fuzz/coinselection.cpp index 331590df7f..644a8dd7ad 100644 --- a/src/wallet/test/fuzz/coinselection.cpp +++ b/src/wallet/test/fuzz/coinselection.cpp @@ -270,7 +270,7 @@ FUZZ_TARGET(coinselection) if (result_srd) { assert(result_srd->GetSelectedValue() >= target); assert(result_srd->GetChange(CHANGE_LOWER, coin_params.m_change_fee) > 0); // Demonstrate that SRD creates change of at least CHANGE_LOWER - result_srd->ComputeAndSetWaste(coin_params.min_viable_change, coin_params.m_cost_of_change, coin_params.m_change_fee); + result_srd->RecalculateWaste(coin_params.min_viable_change, coin_params.m_cost_of_change, coin_params.m_change_fee); (void)result_srd->GetShuffledInputVector(); (void)result_srd->GetInputSet(); } @@ -279,7 +279,7 @@ FUZZ_TARGET(coinselection) auto result_knapsack = KnapsackSolver(group_all, target, change_target, fast_random_context, MAX_STANDARD_TX_WEIGHT); if (result_knapsack) { assert(result_knapsack->GetSelectedValue() >= target); - result_knapsack->ComputeAndSetWaste(coin_params.min_viable_change, coin_params.m_cost_of_change, coin_params.m_change_fee); + result_knapsack->RecalculateWaste(coin_params.min_viable_change, coin_params.m_cost_of_change, coin_params.m_change_fee); (void)result_knapsack->GetShuffledInputVector(); (void)result_knapsack->GetInputSet(); } diff --git a/src/wallet/test/fuzz/crypter.cpp b/src/wallet/test/fuzz/crypter.cpp new file mode 100644 index 0000000000..62dd1bfde0 --- /dev/null +++ b/src/wallet/test/fuzz/crypter.cpp @@ -0,0 +1,92 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/setup_common.h> +#include <wallet/crypter.h> + +namespace wallet { +namespace { + +const TestingSetup* g_setup; +void initialize_crypter() +{ + static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); + g_setup = testing_setup.get(); +} + +FUZZ_TARGET(crypter, .init = initialize_crypter) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + bool good_data{true}; + + CCrypter crypt; + // These values are regularly updated within `CallOneOf` + std::vector<unsigned char> cipher_text_ed; + CKeyingMaterial plain_text_ed; + const std::vector<unsigned char> random_key = ConsumeRandomLengthByteVector(fuzzed_data_provider); + + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 10000) + { + CallOneOf( + fuzzed_data_provider, + [&] { + const std::string random_string = fuzzed_data_provider.ConsumeRandomLengthString(); + SecureString secure_string(random_string.begin(), random_string.end()); + + const unsigned int derivation_method = fuzzed_data_provider.ConsumeBool() ? 0 : fuzzed_data_provider.ConsumeIntegral<unsigned int>(); + + // Limiting the value of nRounds since it is otherwise uselessly expensive and causes a timeout when fuzzing. + crypt.SetKeyFromPassphrase(/*strKeyData=*/secure_string, + /*chSalt=*/ConsumeRandomLengthByteVector(fuzzed_data_provider), + /*nRounds=*/fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, 25000), + /*nDerivationMethod=*/derivation_method); + }, + [&] { + const std::vector<unsigned char> random_vector = ConsumeFixedLengthByteVector(fuzzed_data_provider, 32); + const CKeyingMaterial new_key(random_vector.begin(), random_vector.end()); + const std::vector<unsigned char>& new_IV = ConsumeFixedLengthByteVector(fuzzed_data_provider, 16); + crypt.SetKey(new_key, new_IV); + }, + [&] { + const std::vector<unsigned char> random_vector = ConsumeRandomLengthByteVector(fuzzed_data_provider); + plain_text_ed = CKeyingMaterial(random_vector.begin(), random_vector.end()); + }, + [&] { + cipher_text_ed = ConsumeRandomLengthByteVector(fuzzed_data_provider); + }, + [&] { + (void)crypt.Encrypt(plain_text_ed, cipher_text_ed); + }, + [&] { + (void)crypt.Decrypt(cipher_text_ed, plain_text_ed); + }, + [&] { + const CKeyingMaterial master_key(random_key.begin(), random_key.end()); + const uint256 iv = ConsumeUInt256(fuzzed_data_provider); + EncryptSecret(master_key, plain_text_ed, iv, cipher_text_ed); + }, + [&] { + const CKeyingMaterial master_key(random_key.begin(), random_key.end()); + const uint256 iv = ConsumeUInt256(fuzzed_data_provider); + DecryptSecret(master_key, cipher_text_ed, iv, plain_text_ed); + }, + [&] { + std::optional<CPubKey> random_pub_key = ConsumeDeserializable<CPubKey>(fuzzed_data_provider); + if (!random_pub_key) { + good_data = false; + return; + } + const CPubKey pub_key = *random_pub_key; + const CKeyingMaterial master_key(random_key.begin(), random_key.end()); + const std::vector<unsigned char> crypted_secret = ConsumeRandomLengthByteVector(fuzzed_data_provider); + CKey key; + DecryptKey(master_key, crypted_secret, pub_key, key); + }); + } +} +} // namespace +} // namespace wallet diff --git a/src/wallet/test/fuzz/scriptpubkeyman.cpp b/src/wallet/test/fuzz/scriptpubkeyman.cpp index 228e9629ed..835470aeae 100644 --- a/src/wallet/test/fuzz/scriptpubkeyman.cpp +++ b/src/wallet/test/fuzz/scriptpubkeyman.cpp @@ -137,6 +137,15 @@ FUZZ_TARGET(scriptpubkeyman, .init = initialize_spkm) PKHash{ConsumeUInt160(fuzzed_data_provider)}}; std::string str_sig; (void)spk_manager->SignMessage(msg, pk_hash, str_sig); + (void)spk_manager->GetMetadata(dest); + } + } + }, + [&] { + auto spks{spk_manager->GetScriptPubKeys()}; + for (const CScript& spk : spks) { + if (fuzzed_data_provider.ConsumeBool()) { + spk_manager->MarkUnusedAddresses(spk); } } }, @@ -148,6 +157,10 @@ FUZZ_TARGET(scriptpubkeyman, .init = initialize_spkm) } spk_manager->AddDescriptorKey(key, key.GetPubKey()); spk_manager->TopUp(); + LOCK(spk_manager->cs_desc_man); + auto particular_key{spk_manager->GetKey(key.GetPubKey().GetID())}; + assert(*particular_key == key); + assert(spk_manager->HasPrivKey(key.GetPubKey().GetID())); }, [&] { std::string descriptor; @@ -194,6 +207,9 @@ FUZZ_TARGET(scriptpubkeyman, .init = initialize_spkm) } ); } + + (void)spk_manager->GetEndRange(); + (void)spk_manager->GetKeyPoolSize(); } } // namespace diff --git a/src/wallet/test/fuzz/wallet_bdb_parser.cpp b/src/wallet/test/fuzz/wallet_bdb_parser.cpp index 24ef75f791..6fbd695fc5 100644 --- a/src/wallet/test/fuzz/wallet_bdb_parser.cpp +++ b/src/wallet/test/fuzz/wallet_bdb_parser.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2023 The Bitcoin Core developers +// Copyright (c) 2023-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -52,7 +52,7 @@ FUZZ_TARGET(wallet_bdb_parser, .init = initialize_wallet_bdb_parser) #ifdef USE_BDB bool bdb_ro_err = false; - bool bdb_ro_pgno_err = false; + bool bdb_ro_strict_err = false; #endif auto db{MakeBerkeleyRODatabase(wallet_path, options, status, error)}; if (db) { @@ -61,17 +61,15 @@ FUZZ_TARGET(wallet_bdb_parser, .init = initialize_wallet_bdb_parser) #ifdef USE_BDB bdb_ro_err = true; #endif - if (error.original == "AutoFile::ignore: end of file: iostream error" || - error.original == "AutoFile::read: end of file: iostream error" || + if (error.original.starts_with("AutoFile::ignore: end of file") || + error.original.starts_with("AutoFile::read: end of file") || + error.original.starts_with("AutoFile::seek: ") || error.original == "Not a BDB file" || - error.original == "Unsupported BDB data file version number" || error.original == "Unexpected page type, should be 9 (BTree Metadata)" || error.original == "Unexpected database flags, should only be 0x20 (subdatabases)" || error.original == "Unexpected outer database root page type" || error.original == "Unexpected number of entries in outer database root page" || - error.original == "Subdatabase has an unexpected name" || error.original == "Subdatabase page number has unexpected length" || - error.original == "Unexpected inner database page type" || error.original == "Unknown record type in records page" || error.original == "Unknown record type in internal page" || error.original == "Unexpected page size" || @@ -79,15 +77,21 @@ FUZZ_TARGET(wallet_bdb_parser, .init = initialize_wallet_bdb_parser) error.original == "Page number mismatch" || error.original == "Bad btree level" || error.original == "Bad page size" || - error.original == "File size is not a multiple of page size" || - error.original == "Meta page number mismatch") { + error.original == "Meta page number mismatch" || + error.original == "Data record position not in page" || + error.original == "Internal record position not in page" || + error.original == "LSNs are not reset, this database is not completely flushed. Please reopen then close the database with a version that has BDB support" || + error.original == "Records page has odd number of records" || + error.original == "Bad overflow record page type") { // Do nothing } else if (error.original == "Subdatabase last page is greater than database last page" || error.original == "Page number is greater than database last page" || - error.original == "Page number is greater than subdatabase last page" || - error.original == "Last page number could not fit in file") { + error.original == "Last page number could not fit in file" || + error.original == "Subdatabase has an unexpected name" || + error.original == "Unsupported BDB data file version number" || + error.original == "BDB builtin encryption is not supported") { #ifdef USE_BDB - bdb_ro_pgno_err = true; + bdb_ro_strict_err = true; #endif } else { throw std::runtime_error(error.original); @@ -108,9 +112,8 @@ FUZZ_TARGET(wallet_bdb_parser, .init = initialize_wallet_bdb_parser) return; } assert(db); - if (bdb_ro_pgno_err) { - // BerkeleyRO will throw on opening for errors involving bad page numbers, but BDB does not. - // Ignore those. + if (bdb_ro_strict_err) { + // BerkeleyRO will be stricter than BDB. Ignore when those specific errors are hit. return; } assert(!bdb_ro_err); diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index 9f533bf6ed..b5a3b22c54 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <key_io.h> +#include <node/types.h> #include <util/bip32.h> #include <util/strencodings.h> #include <wallet/wallet.h> @@ -60,7 +61,7 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) // Fill transaction with our data bool complete = true; - BOOST_REQUIRE_EQUAL(TransactionError::OK, m_wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, false, true)); + BOOST_REQUIRE(!m_wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, false, true)); // Get the final tx DataStream ssTx{}; @@ -73,7 +74,7 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) // Try to sign the mutated input SignatureData sigdata; - BOOST_CHECK(m_wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, true, true) != TransactionError::OK); + BOOST_CHECK(m_wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, true, true)); } BOOST_AUTO_TEST_CASE(parse_hd_keypath) diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 3a67b9a433..53f3bcc421 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -235,11 +235,11 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) key.pushKV("scriptPubKey", HexStr(GetScriptForRawPubKey(futureKey.GetPubKey()))); key.pushKV("timestamp", newTip->GetBlockTimeMax() + TIMESTAMP_WINDOW + 1); key.pushKV("internal", UniValue(true)); - keys.push_back(key); + keys.push_back(std::move(key)); JSONRPCRequest request; request.context = &context; request.params.setArray(); - request.params.push_back(keys); + request.params.push_back(std::move(keys)); UniValue response = importmulti().HandleRequest(request); BOOST_CHECK_EQUAL(response.write(), @@ -499,8 +499,10 @@ static void TestWatchOnlyPubKey(LegacyScriptPubKeyMan* spk_man, const CPubKey& a // Cryptographically invalidate a PubKey whilst keeping length and first byte static void PollutePubKey(CPubKey& pubkey) { - std::vector<unsigned char> pubkey_raw(pubkey.begin(), pubkey.end()); - std::fill(pubkey_raw.begin()+1, pubkey_raw.end(), 0); + assert(pubkey.size() >= 1); + std::vector<unsigned char> pubkey_raw; + pubkey_raw.push_back(pubkey[0]); + pubkey_raw.insert(pubkey_raw.end(), pubkey.size() - 1, 0); pubkey = CPubKey(pubkey_raw); assert(!pubkey.IsFullyValid()); assert(pubkey.IsValid()); diff --git a/src/wallet/transaction.h b/src/wallet/transaction.h index 9c27574103..9079f6dd82 100644 --- a/src/wallet/transaction.h +++ b/src/wallet/transaction.h @@ -273,7 +273,7 @@ public: mapValueCopy["fromaccount"] = ""; if (nOrderPos != -1) { - mapValueCopy["n"] = ToString(nOrderPos); + mapValueCopy["n"] = util::ToString(nOrderPos); } if (nTimeSmart) { mapValueCopy["timesmart"] = strprintf("%u", nTimeSmart); diff --git a/src/wallet/types.h b/src/wallet/types.h index 6198f1ae33..7e3b2caeb1 100644 --- a/src/wallet/types.h +++ b/src/wallet/types.h @@ -3,12 +3,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -//! @file Public type definitions that are used inside and outside of the wallet -//! (e.g. by src/wallet and src/interfaces and src/qt code). +//! @file wallet/types.h is a home for public enum and struct type definitions +//! that are used by internally by wallet code, but also used externally by node +//! or GUI code. //! -//! File is home for simple enum and struct definitions that don't deserve -//! separate header files. More complicated wallet public types like -//! CCoinControl that are used externally can have separate headers. +//! This file is intended to define only simple types that do not have external +//! dependencies. More complicated public wallet types like CCoinControl should +//! be defined in dedicated header files. #ifndef BITCOIN_WALLET_TYPES_H #define BITCOIN_WALLET_TYPES_H diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 8a79cf730b..d569c64b43 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -11,7 +11,9 @@ #include <chain.h> #include <coins.h> #include <common/args.h> +#include <common/messages.h> #include <common/settings.h> +#include <common/signmessage.h> #include <common/system.h> #include <consensus/amount.h> #include <consensus/consensus.h> @@ -25,6 +27,7 @@ #include <key.h> #include <key_io.h> #include <logging.h> +#include <node/types.h> #include <outputtype.h> #include <policy/feerate.h> #include <primitives/block.h> @@ -49,10 +52,8 @@ #include <uint256.h> #include <univalue.h> #include <util/check.h> -#include <util/error.h> #include <util/fs.h> #include <util/fs_helpers.h> -#include <util/message.h> #include <util/moneystr.h> #include <util/result.h> #include <util/string.h> @@ -81,7 +82,12 @@ struct KeyOriginInfo; +using common::AmountErrMsg; +using common::AmountHighWarn; +using common::PSBTError; using interfaces::FoundBlock; +using util::ReplaceAll; +using util::ToString; namespace wallet { @@ -1360,13 +1366,13 @@ void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, c } -void CWallet::RecursiveUpdateTxState(const uint256& tx_hash, const TryUpdatingStateFn& try_updating_state) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { +void CWallet::RecursiveUpdateTxState(const uint256& tx_hash, const TryUpdatingStateFn& try_updating_state) { // Do not flush the wallet here for performance reasons WalletBatch batch(GetDatabase(), false); RecursiveUpdateTxState(&batch, tx_hash, try_updating_state); } -void CWallet::RecursiveUpdateTxState(WalletBatch* batch, const uint256& tx_hash, const TryUpdatingStateFn& try_updating_state) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { +void CWallet::RecursiveUpdateTxState(WalletBatch* batch, const uint256& tx_hash, const TryUpdatingStateFn& try_updating_state) { std::set<uint256> todo; std::set<uint256> done; @@ -2172,7 +2178,7 @@ bool CWallet::SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, return false; } -TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& complete, int sighash_type, bool sign, bool bip32derivs, size_t * n_signed, bool finalize) const +std::optional<PSBTError> CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& complete, int sighash_type, bool sign, bool bip32derivs, size_t * n_signed, bool finalize) const { if (n_signed) { *n_signed = 0; @@ -2205,9 +2211,9 @@ TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& comp // Fill in information from ScriptPubKeyMans for (ScriptPubKeyMan* spk_man : GetAllScriptPubKeyMans()) { int n_signed_this_spkm = 0; - TransactionError res = spk_man->FillPSBT(psbtx, txdata, sighash_type, sign, bip32derivs, &n_signed_this_spkm, finalize); - if (res != TransactionError::OK) { - return res; + const auto error{spk_man->FillPSBT(psbtx, txdata, sighash_type, sign, bip32derivs, &n_signed_this_spkm, finalize)}; + if (error) { + return error; } if (n_signed) { @@ -2223,7 +2229,7 @@ TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& comp complete &= PSBTInputSigned(input); } - return TransactionError::OK; + return {}; } SigningResult CWallet::SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 6a998fa398..5bc888462f 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -58,7 +58,9 @@ class Coin; class SigningProvider; enum class MemPoolRemovalReason; enum class SigningResult; -enum class TransactionError; +namespace common { +enum class PSBTError; +} // namespace common namespace interfaces { class Wallet; } @@ -659,7 +661,7 @@ public: * @param[in] finalize whether to create the final scriptSig or scriptWitness if possible * return error */ - TransactionError FillPSBT(PartiallySignedTransaction& psbtx, + std::optional<common::PSBTError> FillPSBT(PartiallySignedTransaction& psbtx, bool& complete, int sighash_type = SIGHASH_DEFAULT, bool sign = true, diff --git a/src/warnings.cpp b/src/warnings.cpp deleted file mode 100644 index 38c0554cf2..0000000000 --- a/src/warnings.cpp +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <config/bitcoin-config.h> // IWYU pragma: keep - -#include <warnings.h> - -#include <common/system.h> -#include <sync.h> -#include <util/translation.h> - -#include <optional> -#include <vector> - -static GlobalMutex g_warnings_mutex; -static bilingual_str g_misc_warnings GUARDED_BY(g_warnings_mutex); -static bool fLargeWorkInvalidChainFound GUARDED_BY(g_warnings_mutex) = false; -static std::optional<bilingual_str> g_timeoffset_warning GUARDED_BY(g_warnings_mutex){}; - -void SetMiscWarning(const bilingual_str& warning) -{ - LOCK(g_warnings_mutex); - g_misc_warnings = warning; -} - -void SetfLargeWorkInvalidChainFound(bool flag) -{ - LOCK(g_warnings_mutex); - fLargeWorkInvalidChainFound = flag; -} - -void SetMedianTimeOffsetWarning(std::optional<bilingual_str> warning) -{ - LOCK(g_warnings_mutex); - g_timeoffset_warning = warning; -} - -std::vector<bilingual_str> GetWarnings() -{ - std::vector<bilingual_str> warnings; - - LOCK(g_warnings_mutex); - - // Pre-release build warning - if (!CLIENT_VERSION_IS_RELEASE) { - warnings.emplace_back(_("This is a pre-release test build - use at your own risk - do not use for mining or merchant applications")); - } - - // Misc warnings like out of disk space and clock is wrong - if (!g_misc_warnings.empty()) { - warnings.emplace_back(g_misc_warnings); - } - - if (fLargeWorkInvalidChainFound) { - warnings.emplace_back(_("Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade.")); - } - - if (g_timeoffset_warning) { - warnings.emplace_back(g_timeoffset_warning.value()); - } - - return warnings; -} diff --git a/src/warnings.h b/src/warnings.h deleted file mode 100644 index 79dc2ffabf..0000000000 --- a/src/warnings.h +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_WARNINGS_H -#define BITCOIN_WARNINGS_H - -#include <optional> -#include <string> -#include <vector> - -struct bilingual_str; - -void SetMiscWarning(const bilingual_str& warning); -void SetfLargeWorkInvalidChainFound(bool flag); -/** Pass std::nullopt to disable the warning */ -void SetMedianTimeOffsetWarning(std::optional<bilingual_str> warning); -/** Return potential problems detected by the node. */ -std::vector<bilingual_str> GetWarnings(); - -#endif // BITCOIN_WARNINGS_H diff --git a/src/zmq/zmqrpc.cpp b/src/zmq/zmqrpc.cpp index fcc2608262..84995d12d3 100644 --- a/src/zmq/zmqrpc.cpp +++ b/src/zmq/zmqrpc.cpp @@ -47,7 +47,7 @@ static RPCHelpMan getzmqnotifications() obj.pushKV("type", n->GetType()); obj.pushKV("address", n->GetAddress()); obj.pushKV("hwm", n->GetOutboundMessageHighWaterMark()); - result.push_back(obj); + result.push_back(std::move(obj)); } } diff --git a/test/functional/feature_addrman.py b/test/functional/feature_addrman.py index 95d33d62ea..2efad70900 100755 --- a/test/functional/feature_addrman.py +++ b/test/functional/feature_addrman.py @@ -6,7 +6,6 @@ import os import re -import struct from test_framework.messages import ser_uint256, hash256, MAGIC_BYTES from test_framework.netutil import ADDRMAN_NEW_BUCKET_COUNT, ADDRMAN_TRIED_BUCKET_COUNT, ADDRMAN_BUCKET_SIZE @@ -28,15 +27,15 @@ def serialize_addrman( tried = [] INCOMPATIBILITY_BASE = 32 r = MAGIC_BYTES[net_magic] - r += struct.pack("B", format) - r += struct.pack("B", INCOMPATIBILITY_BASE + lowest_compatible) + r += format.to_bytes(1, "little") + r += (INCOMPATIBILITY_BASE + lowest_compatible).to_bytes(1, "little") r += ser_uint256(bucket_key) - r += struct.pack("<i", len_new or len(new)) - r += struct.pack("<i", len_tried or len(tried)) + r += (len_new or len(new)).to_bytes(4, "little", signed=True) + r += (len_tried or len(tried)).to_bytes(4, "little", signed=True) ADDRMAN_NEW_BUCKET_COUNT = 1 << 10 - r += struct.pack("<i", ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30)) + r += (ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30)).to_bytes(4, "little", signed=True) for _ in range(ADDRMAN_NEW_BUCKET_COUNT): - r += struct.pack("<i", 0) + r += (0).to_bytes(4, "little", signed=True) checksum = hash256(r) r += mock_checksum or checksum return r diff --git a/test/functional/feature_asmap.py b/test/functional/feature_asmap.py index 024a8fa18c..e469deef49 100755 --- a/test/functional/feature_asmap.py +++ b/test/functional/feature_asmap.py @@ -27,6 +27,7 @@ import os import shutil from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal DEFAULT_ASMAP_FILENAME = 'ip_asn.map' # defined in src/init.cpp ASMAP = '../../src/test/data/asmap.raw' # path to unit test skeleton asmap @@ -118,6 +119,14 @@ class AsmapTest(BitcoinTestFramework): msg = "ASMap Health Check: 4 clearnet peers are mapped to 3 ASNs with 0 peers being unmapped" with self.node.assert_debug_log(expected_msgs=[msg]): self.start_node(0, extra_args=['-asmap']) + raw_addrman = self.node.getrawaddrman() + asns = [] + for _, entries in raw_addrman.items(): + for _, entry in entries.items(): + asn = entry['mapped_as'] + if asn not in asns: + asns.append(asn) + assert_equal(len(asns), 3) os.remove(self.default_asmap) def run_test(self): diff --git a/test/functional/feature_assumeutxo.py b/test/functional/feature_assumeutxo.py index 0d6c92c9fa..658eea0a0e 100755 --- a/test/functional/feature_assumeutxo.py +++ b/test/functional/feature_assumeutxo.py @@ -75,41 +75,76 @@ class AssumeutxoTest(BitcoinTestFramework): with self.nodes[1].assert_debug_log([log_msg]): assert_raises_rpc_error(-32603, f"Unable to load UTXO snapshot{rpc_details}", self.nodes[1].loadtxoutset, bad_snapshot_path) + self.log.info(" - snapshot file with invalid file magic") + parsing_error_code = -22 + bad_magic = 0xf00f00f000 + with open(bad_snapshot_path, 'wb') as f: + f.write(bad_magic.to_bytes(5, "big") + valid_snapshot_contents[5:]) + assert_raises_rpc_error(parsing_error_code, "Unable to parse metadata: Invalid UTXO set snapshot magic bytes. Please check if this is indeed a snapshot file or if you are using an outdated snapshot format.", self.nodes[1].loadtxoutset, bad_snapshot_path) + + self.log.info(" - snapshot file with unsupported version") + for version in [0, 2]: + with open(bad_snapshot_path, 'wb') as f: + f.write(valid_snapshot_contents[:5] + version.to_bytes(2, "little") + valid_snapshot_contents[7:]) + assert_raises_rpc_error(parsing_error_code, f"Unable to parse metadata: Version of snapshot {version} does not match any of the supported versions.", self.nodes[1].loadtxoutset, bad_snapshot_path) + + self.log.info(" - snapshot file with mismatching network magic") + invalid_magics = [ + # magic, name, real + [0xf9beb4d9, "main", True], + [0x0b110907, "test", True], + [0x0a03cf40, "signet", True], + [0x00000000, "", False], + [0xffffffff, "", False], + ] + for [magic, name, real] in invalid_magics: + with open(bad_snapshot_path, 'wb') as f: + f.write(valid_snapshot_contents[:7] + magic.to_bytes(4, 'big') + valid_snapshot_contents[11:]) + if real: + assert_raises_rpc_error(parsing_error_code, f"Unable to parse metadata: The network of the snapshot ({name}) does not match the network of this node (regtest).", self.nodes[1].loadtxoutset, bad_snapshot_path) + else: + assert_raises_rpc_error(parsing_error_code, "Unable to parse metadata: This snapshot has been created for an unrecognized network. This could be a custom signet, a new testnet or possibly caused by data corruption.", self.nodes[1].loadtxoutset, bad_snapshot_path) + self.log.info(" - snapshot file referring to a block that is not in the assumeutxo parameters") prev_block_hash = self.nodes[0].getblockhash(SNAPSHOT_BASE_HEIGHT - 1) bogus_block_hash = "0" * 64 # Represents any unknown block hash + # The height is not used for anything critical currently, so we just + # confirm the manipulation in the error message + bogus_height = 1337 for bad_block_hash in [bogus_block_hash, prev_block_hash]: with open(bad_snapshot_path, 'wb') as f: - # block hash of the snapshot base is stored right at the start (first 32 bytes) - f.write(bytes.fromhex(bad_block_hash)[::-1] + valid_snapshot_contents[32:]) - error_details = f", assumeutxo block hash in snapshot metadata not recognized ({bad_block_hash})" + f.write(valid_snapshot_contents[:11] + bogus_height.to_bytes(4, "little") + bytes.fromhex(bad_block_hash)[::-1] + valid_snapshot_contents[47:]) + error_details = f", assumeutxo block hash in snapshot metadata not recognized (hash: {bad_block_hash}, height: {bogus_height}). The following snapshot heights are available: 110, 299." expected_error(rpc_details=error_details) self.log.info(" - snapshot file with wrong number of coins") - valid_num_coins = int.from_bytes(valid_snapshot_contents[32:32 + 8], "little") + valid_num_coins = int.from_bytes(valid_snapshot_contents[47:47 + 8], "little") for off in [-1, +1]: with open(bad_snapshot_path, 'wb') as f: - f.write(valid_snapshot_contents[:32]) + f.write(valid_snapshot_contents[:47]) f.write((valid_num_coins + off).to_bytes(8, "little")) - f.write(valid_snapshot_contents[32 + 8:]) + f.write(valid_snapshot_contents[47 + 8:]) expected_error(log_msg=f"bad snapshot - coins left over after deserializing 298 coins" if off == -1 else f"bad snapshot format or truncated snapshot after deserializing 299 coins") - self.log.info(" - snapshot file with alternated UTXO data") + self.log.info(" - snapshot file with alternated but parsable UTXO data results in different hash") cases = [ # (content, offset, wrong_hash, custom_message) [b"\xff" * 32, 0, "7d52155c9a9fdc4525b637ef6170568e5dad6fabd0b1fdbb9432010b8453095b", None], # wrong outpoint hash - [(1).to_bytes(4, "little"), 32, "9f4d897031ab8547665b4153317ae2fdbf0130c7840b66427ebc48b881cb80ad", None], # wrong outpoint index - [b"\x81", 36, "3da966ba9826fb6d2604260e01607b55ba44e1a5de298606b08704bc62570ea8", None], # wrong coin code VARINT - [b"\x80", 36, "091e893b3ccb4334378709578025356c8bcb0a623f37c7c4e493133c988648e5", None], # another wrong coin code - [b"\x84\x58", 36, None, "[snapshot] bad snapshot data after deserializing 0 coins"], # wrong coin case with height 364 and coinbase 0 - [b"\xCA\xD2\x8F\x5A", 41, None, "[snapshot] bad snapshot data after deserializing 0 coins - bad tx out value"], # Amount exceeds MAX_MONEY + [(2).to_bytes(1, "little"), 32, None, "[snapshot] bad snapshot data after deserializing 1 coins"], # wrong txid coins count + [b"\xfd\xff\xff", 32, None, "[snapshot] mismatch in coins count in snapshot metadata and actual snapshot data"], # txid coins count exceeds coins left + [b"\x01", 33, "9f4d897031ab8547665b4153317ae2fdbf0130c7840b66427ebc48b881cb80ad", None], # wrong outpoint index + [b"\x81", 34, "3da966ba9826fb6d2604260e01607b55ba44e1a5de298606b08704bc62570ea8", None], # wrong coin code VARINT + [b"\x80", 34, "091e893b3ccb4334378709578025356c8bcb0a623f37c7c4e493133c988648e5", None], # another wrong coin code + [b"\x84\x58", 34, None, "[snapshot] bad snapshot data after deserializing 0 coins"], # wrong coin case with height 364 and coinbase 0 + [b"\xCA\xD2\x8F\x5A", 39, None, "[snapshot] bad snapshot data after deserializing 0 coins - bad tx out value"], # Amount exceeds MAX_MONEY ] for content, offset, wrong_hash, custom_message in cases: with open(bad_snapshot_path, "wb") as f: - f.write(valid_snapshot_contents[:(32 + 8 + offset)]) + # Prior to offset: Snapshot magic, snapshot version, network magic, height, hash, coins count + f.write(valid_snapshot_contents[:(5 + 2 + 4 + 4 + 32 + 8 + offset)]) f.write(content) - f.write(valid_snapshot_contents[(32 + 8 + offset + len(content)):]) + f.write(valid_snapshot_contents[(5 + 2 + 4 + 4 + 32 + 8 + offset + len(content)):]) log_msg = custom_message if custom_message is not None else f"[snapshot] bad snapshot content hash: expected a4bf3407ccb2cc0145c49ebba8fa91199f8a3903daf0883875941497d2493c27, got {wrong_hash}" expected_error(log_msg=log_msg) @@ -161,6 +196,14 @@ class AssumeutxoTest(BitcoinTestFramework): path = node.datadir_path / node.chain / "invalid" / "path" assert_raises_rpc_error(-8, "Couldn't open file {} for reading.".format(path), node.loadtxoutset, path) + def test_snapshot_with_less_work(self, dump_output_path): + self.log.info("Test bitcoind should fail when snapshot has less accumulated work than this node.") + node = self.nodes[0] + assert_equal(node.getblockcount(), FINAL_HEIGHT) + with node.assert_debug_log(expected_msgs=["[snapshot] activation failed - work does not exceed active chainstate"]): + assert_raises_rpc_error(-32603, "Unable to load UTXO snapshot", node.loadtxoutset, dump_output_path) + self.restart_node(0, extra_args=self.extra_args[0]) + def run_test(self): """ Bring up two (disconnected) nodes, mine some new blocks on the first, @@ -242,6 +285,7 @@ class AssumeutxoTest(BitcoinTestFramework): assert_equal(n0.getblockchaininfo()["blocks"], FINAL_HEIGHT) + self.test_snapshot_with_less_work(dump_output['path']) self.test_invalid_mempool_state(dump_output['path']) self.test_invalid_snapshot_scenarios(dump_output['path']) self.test_invalid_chainstate_scenarios() diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index 8768d4040d..14b92d6733 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -78,8 +78,8 @@ class BIP68Test(BitcoinTestFramework): self.log.info("Activating BIP68 (and 112/113)") self.activateCSV() - self.log.info("Verifying nVersion=2 transactions are standard.") - self.log.info("Note that nVersion=2 transactions are always standard (independent of BIP68 activation status).") + self.log.info("Verifying version=2 transactions are standard.") + self.log.info("Note that version=2 transactions are always standard (independent of BIP68 activation status).") self.test_version2_relay() self.log.info("Passed") @@ -107,7 +107,7 @@ class BIP68Test(BitcoinTestFramework): # This transaction will enable sequence-locks, so this transaction should # fail tx2 = CTransaction() - tx2.nVersion = 2 + tx2.version = 2 sequence_value = sequence_value & 0x7fffffff tx2.vin = [CTxIn(COutPoint(tx1_id, 0), nSequence=sequence_value)] tx2.wit.vtxinwit = [CTxInWitness()] @@ -119,7 +119,7 @@ class BIP68Test(BitcoinTestFramework): # Setting the version back down to 1 should disable the sequence lock, # so this should be accepted. - tx2.nVersion = 1 + tx2.version = 1 self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=tx2.serialize().hex()) @@ -159,7 +159,7 @@ class BIP68Test(BitcoinTestFramework): using_sequence_locks = False tx = CTransaction() - tx.nVersion = 2 + tx.version = 2 value = 0 for j in range(num_inputs): sequence_value = 0xfffffffe # this disables sequence locks @@ -228,7 +228,7 @@ class BIP68Test(BitcoinTestFramework): # Anyone-can-spend mempool tx. # Sequence lock of 0 should pass. tx2 = CTransaction() - tx2.nVersion = 2 + tx2.version = 2 tx2.vin = [CTxIn(COutPoint(tx1.sha256, 0), nSequence=0)] tx2.vout = [CTxOut(int(tx1.vout[0].nValue - self.relayfee * COIN), SCRIPT_W0_SH_OP_TRUE)] self.wallet.sign_tx(tx=tx2) @@ -246,7 +246,7 @@ class BIP68Test(BitcoinTestFramework): sequence_value |= SEQUENCE_LOCKTIME_TYPE_FLAG tx = CTransaction() - tx.nVersion = 2 + tx.version = 2 tx.vin = [CTxIn(COutPoint(orig_tx.sha256, 0), nSequence=sequence_value)] tx.wit.vtxinwit = [CTxInWitness()] tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] @@ -360,7 +360,7 @@ class BIP68Test(BitcoinTestFramework): # Make an anyone-can-spend transaction tx2 = CTransaction() - tx2.nVersion = 1 + tx2.version = 1 tx2.vin = [CTxIn(COutPoint(tx1.sha256, 0), nSequence=0)] tx2.vout = [CTxOut(int(tx1.vout[0].nValue - self.relayfee * COIN), SCRIPT_W0_SH_OP_TRUE)] @@ -376,7 +376,7 @@ class BIP68Test(BitcoinTestFramework): sequence_value = 100 # 100 block relative locktime tx3 = CTransaction() - tx3.nVersion = 2 + tx3.version = 2 tx3.vin = [CTxIn(COutPoint(tx2.sha256, 0), nSequence=sequence_value)] tx3.wit.vtxinwit = [CTxInWitness()] tx3.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index 8a95975184..932f37a083 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test block processing.""" import copy -import struct import time from test_framework.blocktools import ( @@ -67,7 +66,7 @@ class CBrokenBlock(CBlock): def serialize(self, with_witness=False): r = b"" r += super(CBlock, self).serialize() - r += struct.pack("<BQ", 255, len(self.vtx)) + r += (255).to_bytes(1, "little") + len(self.vtx).to_bytes(8, "little") for tx in self.vtx: if with_witness: r += tx.serialize_with_witness() diff --git a/test/functional/feature_csv_activation.py b/test/functional/feature_csv_activation.py index bc1f9e8f2f..2db9682931 100755 --- a/test/functional/feature_csv_activation.py +++ b/test/functional/feature_csv_activation.py @@ -110,7 +110,7 @@ class BIP68_112_113Test(BitcoinTestFramework): def create_bip112special(self, input, txversion): tx = self.create_self_transfer_from_utxo(input) - tx.nVersion = txversion + tx.version = txversion self.miniwallet.sign_tx(tx) tx.vin[0].scriptSig = CScript([-1, OP_CHECKSEQUENCEVERIFY, OP_DROP] + list(CScript(tx.vin[0].scriptSig))) tx.rehash() @@ -118,7 +118,7 @@ class BIP68_112_113Test(BitcoinTestFramework): def create_bip112emptystack(self, input, txversion): tx = self.create_self_transfer_from_utxo(input) - tx.nVersion = txversion + tx.version = txversion self.miniwallet.sign_tx(tx) tx.vin[0].scriptSig = CScript([OP_CHECKSEQUENCEVERIFY] + list(CScript(tx.vin[0].scriptSig))) tx.rehash() @@ -136,7 +136,7 @@ class BIP68_112_113Test(BitcoinTestFramework): for i, (sdf, srhb, stf, srlb) in enumerate(product(*[[True, False]] * 4)): locktime = relative_locktime(sdf, srhb, stf, srlb) tx = self.create_self_transfer_from_utxo(bip68inputs[i]) - tx.nVersion = txversion + tx.version = txversion tx.vin[0].nSequence = locktime + locktime_delta self.miniwallet.sign_tx(tx) txs.append({'tx': tx, 'sdf': sdf, 'stf': stf}) @@ -154,7 +154,7 @@ class BIP68_112_113Test(BitcoinTestFramework): tx.vin[0].nSequence = BASE_RELATIVE_LOCKTIME + locktime_delta else: # vary nSequence instead, OP_CSV is fixed tx.vin[0].nSequence = locktime + locktime_delta - tx.nVersion = txversion + tx.version = txversion self.miniwallet.sign_tx(tx) if varyOP_CSV: tx.vin[0].scriptSig = CScript([locktime, OP_CHECKSEQUENCEVERIFY, OP_DROP] + list(CScript(tx.vin[0].scriptSig))) @@ -257,10 +257,10 @@ class BIP68_112_113Test(BitcoinTestFramework): # BIP113 test transaction will be modified before each use to put in appropriate block time bip113tx_v1 = self.create_self_transfer_from_utxo(bip113input) bip113tx_v1.vin[0].nSequence = 0xFFFFFFFE - bip113tx_v1.nVersion = 1 + bip113tx_v1.version = 1 bip113tx_v2 = self.create_self_transfer_from_utxo(bip113input) bip113tx_v2.vin[0].nSequence = 0xFFFFFFFE - bip113tx_v2.nVersion = 2 + bip113tx_v2.version = 2 # For BIP68 test all 16 relative sequence locktimes bip68txs_v1 = self.create_bip68txs(bip68inputs, 1) diff --git a/test/functional/feature_framework_miniwallet.py b/test/functional/feature_framework_miniwallet.py new file mode 100755 index 0000000000..f108289018 --- /dev/null +++ b/test/functional/feature_framework_miniwallet.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test MiniWallet.""" +from test_framework.blocktools import COINBASE_MATURITY +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_greater_than_or_equal, +) +from test_framework.wallet import ( + MiniWallet, + MiniWalletMode, +) + + +class FeatureFrameworkMiniWalletTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def test_tx_padding(self): + """Verify that MiniWallet's transaction padding (`target_weight` parameter) + works accurately enough (i.e. at most 3 WUs higher) with all modes.""" + for mode_name, wallet in self.wallets: + self.log.info(f"Test tx padding with MiniWallet mode {mode_name}...") + utxo = wallet.get_utxo(mark_as_spent=False) + for target_weight in [1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000, 4000000, + 989, 2001, 4337, 13371, 23219, 49153, 102035, 223419, 3999989]: + tx = wallet.create_self_transfer(utxo_to_spend=utxo, target_weight=target_weight)["tx"] + self.log.debug(f"-> target weight: {target_weight}, actual weight: {tx.get_weight()}") + assert_greater_than_or_equal(tx.get_weight(), target_weight) + assert_greater_than_or_equal(target_weight + 3, tx.get_weight()) + + def run_test(self): + node = self.nodes[0] + self.wallets = [ + ("ADDRESS_OP_TRUE", MiniWallet(node, mode=MiniWalletMode.ADDRESS_OP_TRUE)), + ("RAW_OP_TRUE", MiniWallet(node, mode=MiniWalletMode.RAW_OP_TRUE)), + ("RAW_P2PK", MiniWallet(node, mode=MiniWalletMode.RAW_P2PK)), + ] + for _, wallet in self.wallets: + self.generate(wallet, 10) + self.generate(wallet, COINBASE_MATURITY) + + self.test_tx_padding() + + +if __name__ == '__main__': + FeatureFrameworkMiniWalletTest().main() diff --git a/test/functional/feature_framework_unit_tests.py b/test/functional/feature_framework_unit_tests.py index f03f084bed..14d83f8a70 100755 --- a/test/functional/feature_framework_unit_tests.py +++ b/test/functional/feature_framework_unit_tests.py @@ -27,6 +27,7 @@ TEST_FRAMEWORK_MODULES = [ "crypto.ripemd160", "crypto.secp256k1", "script", + "script_util", "segwit_addr", "wallet_util", ] diff --git a/test/functional/feature_reindex.py b/test/functional/feature_reindex.py index f0f32a61ab..835cd0c5cf 100755 --- a/test/functional/feature_reindex.py +++ b/test/functional/feature_reindex.py @@ -73,6 +73,25 @@ class ReindexTest(BitcoinTestFramework): # All blocks should be accepted and processed. assert_equal(self.nodes[0].getblockcount(), 12) + def continue_reindex_after_shutdown(self): + node = self.nodes[0] + self.generate(node, 1500) + + # Restart node with reindex and stop reindex as soon as it starts reindexing + self.log.info("Restarting node while reindexing..") + node.stop_node() + with node.busy_wait_for_debug_log([b'initload thread start']): + node.start(['-blockfilterindex', '-reindex']) + node.wait_for_rpc_connection(wait_for_import=False) + node.stop_node() + + # Start node without the reindex flag and verify it does not wipe the indexes data again + db_path = node.chain_path / 'indexes' / 'blockfilter' / 'basic' / 'db' + with node.assert_debug_log(expected_msgs=[f'Opening LevelDB in {db_path}'], unexpected_msgs=[f'Wiping LevelDB in {db_path}']): + node.start(['-blockfilterindex']) + node.wait_for_rpc_connection(wait_for_import=False) + node.stop_node() + def run_test(self): self.reindex(False) self.reindex(True) @@ -80,6 +99,7 @@ class ReindexTest(BitcoinTestFramework): self.reindex(True) self.out_of_order() + self.continue_reindex_after_shutdown() if __name__ == '__main__': diff --git a/test/functional/feature_settings.py b/test/functional/feature_settings.py index 0214e781de..1cd0aeabd3 100755 --- a/test/functional/feature_settings.py +++ b/test/functional/feature_settings.py @@ -25,7 +25,7 @@ class SettingsTest(BitcoinTestFramework): # Assert default settings file was created self.stop_node(0) - default_settings = {"_warning_": "This file is automatically generated and updated by Bitcoin Core. Please do not edit this file while the node is running, as any changes might be ignored or overwritten."} + default_settings = {"_warning_": f"This file is automatically generated and updated by {self.config['environment']['PACKAGE_NAME']}. Please do not edit this file while the node is running, as any changes might be ignored or overwritten."} with settings.open() as fp: assert_equal(json.load(fp), default_settings) diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index e7d65b4539..1a0844d240 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -1408,10 +1408,10 @@ class TaprootTest(BitcoinTestFramework): left = done while left: - # Construct CTransaction with random nVersion, nLocktime + # Construct CTransaction with random version, nLocktime tx = CTransaction() - tx.nVersion = random.choice([1, 2, random.randint(-0x80000000, 0x7fffffff)]) - min_sequence = (tx.nVersion != 1 and tx.nVersion != 0) * 0x80000000 # The minimum sequence number to disable relative locktime + tx.version = random.choice([1, 2, random.getrandbits(32)]) + min_sequence = (tx.version != 1 and tx.version != 0) * 0x80000000 # The minimum sequence number to disable relative locktime if random.choice([True, False]): tx.nLockTime = random.randrange(LOCKTIME_THRESHOLD, self.lastblocktime - 7200) # all absolute locktimes in the past else: @@ -1502,8 +1502,8 @@ class TaprootTest(BitcoinTestFramework): is_standard_tx = ( fail_input is None # Must be valid to be standard and (all(utxo.spender.is_standard for utxo in input_utxos)) # All inputs must be standard - and tx.nVersion >= 1 # The tx version must be standard - and tx.nVersion <= 2) + and tx.version >= 1 # The tx version must be standard + and tx.version <= 2) tx.rehash() msg = ','.join(utxo.spender.comment + ("*" if n == fail_input else "") for n, utxo in enumerate(input_utxos)) if is_standard_tx: @@ -1530,7 +1530,7 @@ class TaprootTest(BitcoinTestFramework): # Deterministically mine coins to OP_TRUE in block 1 assert_equal(self.nodes[0].getblockcount(), 0) coinbase = CTransaction() - coinbase.nVersion = 1 + coinbase.version = 1 coinbase.vin = [CTxIn(COutPoint(0, 0xffffffff), CScript([OP_1, OP_1]), SEQUENCE_FINAL)] coinbase.vout = [CTxOut(5000000000, CScript([OP_1]))] coinbase.nLockTime = 0 @@ -1622,7 +1622,7 @@ class TaprootTest(BitcoinTestFramework): for i, spk in enumerate(old_spks + tap_spks): val = 42000000 * (i + 7) tx = CTransaction() - tx.nVersion = 1 + tx.version = 1 tx.vin = [CTxIn(COutPoint(lasttxid, i & 1), CScript([]), SEQUENCE_FINAL)] tx.vout = [CTxOut(val, spk), CTxOut(amount - val, CScript([OP_1]))] if i & 1: @@ -1679,7 +1679,7 @@ class TaprootTest(BitcoinTestFramework): # Construct a deterministic transaction spending all outputs created above. tx = CTransaction() - tx.nVersion = 2 + tx.version = 2 tx.vin = [] inputs = [] input_spks = [tap_spks[0], tap_spks[1], old_spks[0], tap_spks[2], tap_spks[5], old_spks[2], tap_spks[6], tap_spks[3], tap_spks[4]] diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index 83bb5121e5..a6628dcbf3 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -8,6 +8,7 @@ from decimal import Decimal import re from test_framework.blocktools import COINBASE_MATURITY +from test_framework.netutil import test_ipv6_local from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -15,6 +16,7 @@ from test_framework.util import ( assert_raises_process_error, assert_raises_rpc_error, get_auth_cookie, + rpc_port, ) import time @@ -107,6 +109,53 @@ class TestBitcoinCli(BitcoinTestFramework): self.log.info("Test connecting to a non-existing server") assert_raises_process_error(1, "Could not connect to the server", self.nodes[0].cli('-rpcport=1').echo) + self.log.info("Test handling of invalid ports in rpcconnect") + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: 127.0.0.1:notaport", self.nodes[0].cli("-rpcconnect=127.0.0.1:notaport").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: 127.0.0.1:-1", self.nodes[0].cli("-rpcconnect=127.0.0.1:-1").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: 127.0.0.1:0", self.nodes[0].cli("-rpcconnect=127.0.0.1:0").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: 127.0.0.1:65536", self.nodes[0].cli("-rpcconnect=127.0.0.1:65536").echo) + + self.log.info("Checking for IPv6") + have_ipv6 = test_ipv6_local() + if not have_ipv6: + self.log.info("Skipping IPv6 tests") + + if have_ipv6: + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: [::1]:notaport", self.nodes[0].cli("-rpcconnect=[::1]:notaport").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: [::1]:-1", self.nodes[0].cli("-rpcconnect=[::1]:-1").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: [::1]:0", self.nodes[0].cli("-rpcconnect=[::1]:0").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: [::1]:65536", self.nodes[0].cli("-rpcconnect=[::1]:65536").echo) + + self.log.info("Test handling of invalid ports in rpcport") + assert_raises_process_error(1, "Invalid port provided in -rpcport: notaport", self.nodes[0].cli("-rpcport=notaport").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcport: -1", self.nodes[0].cli("-rpcport=-1").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcport: 0", self.nodes[0].cli("-rpcport=0").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcport: 65536", self.nodes[0].cli("-rpcport=65536").echo) + + self.log.info("Test port usage preferences") + node_rpc_port = rpc_port(self.nodes[0].index) + # Prevent bitcoin-cli from using existing rpcport in conf + conf_rpcport = "rpcport=" + str(node_rpc_port) + self.nodes[0].replace_in_config([(conf_rpcport, "#" + conf_rpcport)]) + # prefer rpcport over rpcconnect + assert_raises_process_error(1, "Could not connect to the server 127.0.0.1:1", self.nodes[0].cli(f"-rpcconnect=127.0.0.1:{node_rpc_port}", "-rpcport=1").echo) + if have_ipv6: + assert_raises_process_error(1, "Could not connect to the server ::1:1", self.nodes[0].cli(f"-rpcconnect=[::1]:{node_rpc_port}", "-rpcport=1").echo) + + assert_equal(BLOCKS, self.nodes[0].cli("-rpcconnect=127.0.0.1:18999", f'-rpcport={node_rpc_port}').getblockcount()) + if have_ipv6: + assert_equal(BLOCKS, self.nodes[0].cli("-rpcconnect=[::1]:18999", f'-rpcport={node_rpc_port}').getblockcount()) + + # prefer rpcconnect port over default + assert_equal(BLOCKS, self.nodes[0].cli(f"-rpcconnect=127.0.0.1:{node_rpc_port}").getblockcount()) + if have_ipv6: + assert_equal(BLOCKS, self.nodes[0].cli(f"-rpcconnect=[::1]:{node_rpc_port}").getblockcount()) + + # prefer rpcport over default + assert_equal(BLOCKS, self.nodes[0].cli(f'-rpcport={node_rpc_port}').getblockcount()) + # Re-enable rpcport in conf if present + self.nodes[0].replace_in_config([("#" + conf_rpcport, conf_rpcport)]) + self.log.info("Test connecting with non-existing RPC cookie file") assert_raises_process_error(1, "Could not locate RPC credentials", self.nodes[0].cli('-rpccookiefile=does-not-exist', '-rpcpassword=').echo) diff --git a/test/functional/interface_rpc.py b/test/functional/interface_rpc.py index b08ca42796..6c1855c400 100755 --- a/test/functional/interface_rpc.py +++ b/test/functional/interface_rpc.py @@ -14,7 +14,6 @@ from typing import Optional import subprocess -RPC_INVALID_ADDRESS_OR_KEY = -5 RPC_INVALID_PARAMETER = -8 RPC_METHOD_NOT_FOUND = -32601 RPC_INVALID_REQUEST = -32600 diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index b00be5f4f0..e1cee46839 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -18,6 +18,7 @@ from test_framework.messages import ( CTxInWitness, CTxOut, MAX_BLOCK_WEIGHT, + WITNESS_SCALE_FACTOR, MAX_MONEY, SEQUENCE_FINAL, tx_from_hex, @@ -228,7 +229,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.log.info('A really large transaction') tx = tx_from_hex(raw_tx_reference) - tx.vin = [tx.vin[0]] * math.ceil(MAX_BLOCK_WEIGHT // 4 / len(tx.vin[0].serialize())) + tx.vin = [tx.vin[0]] * math.ceil((MAX_BLOCK_WEIGHT // WITNESS_SCALE_FACTOR) / len(tx.vin[0].serialize())) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'bad-txns-oversize'}], rawtxs=[tx.serialize().hex()], @@ -287,7 +288,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.log.info('Some nonstandard transactions') tx = tx_from_hex(raw_tx_reference) - tx.nVersion = 3 # A version currently non-standard + tx.version = 4 # A version currently non-standard self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'version'}], rawtxs=[tx.serialize().hex()], diff --git a/test/functional/mempool_accept_v3.py b/test/functional/mempool_accept_v3.py index 8285b82c19..d4a33c232e 100755 --- a/test/functional/mempool_accept_v3.py +++ b/test/functional/mempool_accept_v3.py @@ -6,6 +6,7 @@ from decimal import Decimal from test_framework.messages import ( MAX_BIP125_RBF_SEQUENCE, + WITNESS_SCALE_FACTOR, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -21,6 +22,7 @@ from test_framework.wallet import ( ) MAX_REPLACEMENT_CANDIDATES = 100 +V3_MAX_VSIZE = 10000 def cleanup(extra_args=None): def decorator(func): @@ -40,7 +42,7 @@ def cleanup(extra_args=None): class MempoolAcceptV3(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.extra_args = [["-acceptnonstdtxn=1"]] + self.extra_args = [[]] self.setup_clean_chain = True def check_mempool(self, txids): @@ -49,7 +51,21 @@ class MempoolAcceptV3(BitcoinTestFramework): assert_equal(len(txids), len(mempool_contents)) assert all([txid in txids for txid in mempool_contents]) - @cleanup(extra_args=["-datacarriersize=1000", "-acceptnonstdtxn=1"]) + @cleanup(extra_args=["-datacarriersize=20000"]) + def test_v3_max_vsize(self): + node = self.nodes[0] + self.log.info("Test v3-specific maximum transaction vsize") + tx_v3_heavy = self.wallet.create_self_transfer(target_weight=(V3_MAX_VSIZE + 1) * WITNESS_SCALE_FACTOR, version=3) + assert_greater_than_or_equal(tx_v3_heavy["tx"].get_vsize(), V3_MAX_VSIZE) + expected_error_heavy = f"v3-rule-violation, v3 tx {tx_v3_heavy['txid']} (wtxid={tx_v3_heavy['wtxid']}) is too big" + assert_raises_rpc_error(-26, expected_error_heavy, node.sendrawtransaction, tx_v3_heavy["hex"]) + self.check_mempool([]) + + # Ensure we are hitting the v3-specific limit and not something else + tx_v2_heavy = self.wallet.send_self_transfer(from_node=node, target_weight=(V3_MAX_VSIZE + 1) * WITNESS_SCALE_FACTOR, version=2) + self.check_mempool([tx_v2_heavy["txid"]]) + + @cleanup(extra_args=["-datacarriersize=1000"]) def test_v3_acceptance(self): node = self.nodes[0] self.log.info("Test a child of a v3 transaction cannot be more than 1000vB") @@ -89,7 +105,7 @@ class MempoolAcceptV3(BitcoinTestFramework): self.check_mempool([tx_v3_parent_normal["txid"], tx_v3_child_almost_heavy_rbf["txid"]]) assert_equal(node.getmempoolentry(tx_v3_parent_normal["txid"])["descendantcount"], 2) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) + @cleanup(extra_args=None) def test_v3_replacement(self): node = self.nodes[0] self.log.info("Test v3 transactions may be replaced by v3 transactions") @@ -146,7 +162,7 @@ class MempoolAcceptV3(BitcoinTestFramework): self.check_mempool([tx_v3_bip125_rbf_v2["txid"], tx_v3_parent["txid"], tx_v3_child["txid"]]) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) + @cleanup(extra_args=None) def test_v3_bip125(self): node = self.nodes[0] self.log.info("Test v3 transactions that don't signal BIP125 are replaceable") @@ -170,7 +186,7 @@ class MempoolAcceptV3(BitcoinTestFramework): ) self.check_mempool([tx_v3_no_bip125_rbf["txid"]]) - @cleanup(extra_args=["-datacarriersize=40000", "-acceptnonstdtxn=1"]) + @cleanup(extra_args=["-datacarriersize=40000"]) def test_v3_reorg(self): node = self.nodes[0] self.log.info("Test that, during a reorg, v3 rules are not enforced") @@ -192,7 +208,7 @@ class MempoolAcceptV3(BitcoinTestFramework): node.reconsiderblock(block[0]) - @cleanup(extra_args=["-limitdescendantsize=10", "-datacarriersize=40000", "-acceptnonstdtxn=1"]) + @cleanup(extra_args=["-limitdescendantsize=10", "-datacarriersize=40000"]) def test_nondefault_package_limits(self): """ Max standard tx size + v3 rules imply the ancestor/descendant rules (at their default @@ -201,25 +217,51 @@ class MempoolAcceptV3(BitcoinTestFramework): """ node = self.nodes[0] self.log.info("Test that a decreased limitdescendantsize also applies to v3 child") - tx_v3_parent_large1 = self.wallet.send_self_transfer(from_node=node, target_weight=99900, version=3) - tx_v3_child_large1 = self.wallet.create_self_transfer(utxo_to_spend=tx_v3_parent_large1["new_utxo"], version=3) - # Child is within v3 limits, but parent's descendant limit is exceeded - assert_greater_than(1000, tx_v3_child_large1["tx"].get_vsize()) + parent_target_weight = 9990 * WITNESS_SCALE_FACTOR + child_target_weight = 500 * WITNESS_SCALE_FACTOR + tx_v3_parent_large1 = self.wallet.send_self_transfer( + from_node=node, + target_weight=parent_target_weight, + version=3 + ) + tx_v3_child_large1 = self.wallet.create_self_transfer( + utxo_to_spend=tx_v3_parent_large1["new_utxo"], + target_weight=child_target_weight, + version=3 + ) + + # Parent and child are within v3 limits, but parent's 10kvB descendant limit is exceeded + assert_greater_than_or_equal(V3_MAX_VSIZE, tx_v3_parent_large1["tx"].get_vsize()) + assert_greater_than_or_equal(1000, tx_v3_child_large1["tx"].get_vsize()) + assert_greater_than(tx_v3_parent_large1["tx"].get_vsize() + tx_v3_child_large1["tx"].get_vsize(), 10000) + assert_raises_rpc_error(-26, f"too-long-mempool-chain, exceeds descendant size limit for tx {tx_v3_parent_large1['txid']}", node.sendrawtransaction, tx_v3_child_large1["hex"]) self.check_mempool([tx_v3_parent_large1["txid"]]) assert_equal(node.getmempoolentry(tx_v3_parent_large1["txid"])["descendantcount"], 1) self.generate(node, 1) self.log.info("Test that a decreased limitancestorsize also applies to v3 parent") - self.restart_node(0, extra_args=["-limitancestorsize=10", "-datacarriersize=40000", "-acceptnonstdtxn=1"]) - tx_v3_parent_large2 = self.wallet.send_self_transfer(from_node=node, target_weight=99900, version=3) - tx_v3_child_large2 = self.wallet.create_self_transfer(utxo_to_spend=tx_v3_parent_large2["new_utxo"], version=3) - # Child is within v3 limits + self.restart_node(0, extra_args=["-limitancestorsize=10", "-datacarriersize=40000"]) + tx_v3_parent_large2 = self.wallet.send_self_transfer( + from_node=node, + target_weight=parent_target_weight, + version=3 + ) + tx_v3_child_large2 = self.wallet.create_self_transfer( + utxo_to_spend=tx_v3_parent_large2["new_utxo"], + target_weight=child_target_weight, + version=3 + ) + + # Parent and child are within v3 limits + assert_greater_than_or_equal(V3_MAX_VSIZE, tx_v3_parent_large2["tx"].get_vsize()) assert_greater_than_or_equal(1000, tx_v3_child_large2["tx"].get_vsize()) + assert_greater_than(tx_v3_parent_large2["tx"].get_vsize() + tx_v3_child_large2["tx"].get_vsize(), 10000) + assert_raises_rpc_error(-26, f"too-long-mempool-chain, exceeds ancestor size limit", node.sendrawtransaction, tx_v3_child_large2["hex"]) self.check_mempool([tx_v3_parent_large2["txid"]]) - @cleanup(extra_args=["-datacarriersize=1000", "-acceptnonstdtxn=1"]) + @cleanup(extra_args=["-datacarriersize=1000"]) def test_v3_ancestors_package(self): self.log.info("Test that v3 ancestor limits are checked within the package") node = self.nodes[0] @@ -262,7 +304,7 @@ class MempoolAcceptV3(BitcoinTestFramework): result = node.testmempoolaccept([tx_v3_parent["hex"], tx_v3_child["hex"], tx_v3_grandchild["hex"]]) assert all([txresult["package-error"] == f"v3-violation, tx {tx_v3_grandchild['txid']} (wtxid={tx_v3_grandchild['wtxid']}) would have too many ancestors" for txresult in result]) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) + @cleanup(extra_args=None) def test_v3_ancestors_package_and_mempool(self): """ A v3 transaction in a package cannot have 2 v3 parents. @@ -292,7 +334,7 @@ class MempoolAcceptV3(BitcoinTestFramework): assert_equal(result['package_msg'], f"v3-violation, tx {tx_child_violator['txid']} (wtxid={tx_child_violator['wtxid']}) would have too many ancestors") self.check_mempool([tx_in_mempool["txid"]]) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) + @cleanup(extra_args=None) def test_sibling_eviction_package(self): """ When a transaction has a mempool sibling, it may be eligible for sibling eviction. @@ -368,7 +410,7 @@ class MempoolAcceptV3(BitcoinTestFramework): assert_equal(result_package_cpfp["tx-results"][tx_sibling_3['wtxid']]['error'], expected_error_cpfp) - @cleanup(extra_args=["-datacarriersize=1000", "-acceptnonstdtxn=1"]) + @cleanup(extra_args=["-datacarriersize=1000"]) def test_v3_package_inheritance(self): self.log.info("Test that v3 inheritance is checked within package") node = self.nodes[0] @@ -387,7 +429,7 @@ class MempoolAcceptV3(BitcoinTestFramework): assert_equal(result['package_msg'], f"v3-violation, non-v3 tx {tx_v2_child['txid']} (wtxid={tx_v2_child['wtxid']}) cannot spend from v3 tx {tx_v3_parent['txid']} (wtxid={tx_v3_parent['wtxid']})") self.check_mempool([]) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) + @cleanup(extra_args=None) def test_v3_in_testmempoolaccept(self): node = self.nodes[0] @@ -437,7 +479,7 @@ class MempoolAcceptV3(BitcoinTestFramework): test_accept_2children_with_in_mempool_parent = node.testmempoolaccept([tx_v3_child_1["hex"], tx_v3_child_2["hex"]]) assert all([result["package-error"] == expected_error_extra for result in test_accept_2children_with_in_mempool_parent]) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) + @cleanup(extra_args=None) def test_reorg_2child_rbf(self): node = self.nodes[0] self.log.info("Test that children of a v3 transaction can be replaced individually, even if there are multiple due to reorg") @@ -468,7 +510,7 @@ class MempoolAcceptV3(BitcoinTestFramework): self.check_mempool([ancestor_tx["txid"], child_1_conflict["txid"], child_2["txid"]]) assert_equal(node.getmempoolentry(ancestor_tx["txid"])["descendantcount"], 3) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) + @cleanup(extra_args=None) def test_v3_sibling_eviction(self): self.log.info("Test sibling eviction for v3") node = self.nodes[0] @@ -541,7 +583,7 @@ class MempoolAcceptV3(BitcoinTestFramework): node.sendrawtransaction(tx_v3_child_3["hex"]) self.check_mempool(txids_v2_100 + [tx_v3_parent["txid"], tx_v3_child_3["txid"]]) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) + @cleanup(extra_args=None) def test_reorg_sibling_eviction_1p2c(self): node = self.nodes[0] self.log.info("Test that sibling eviction is not allowed when multiple siblings exist") @@ -585,6 +627,7 @@ class MempoolAcceptV3(BitcoinTestFramework): node = self.nodes[0] self.wallet = MiniWallet(node) self.generate(self.wallet, 120) + self.test_v3_max_vsize() self.test_v3_acceptance() self.test_v3_replacement() self.test_v3_bip125() diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index d46924f4ce..49a0a32c45 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -59,7 +59,7 @@ class MempoolLimitTest(BitcoinTestFramework): mempoolmin_feerate = node.getmempoolinfo()["mempoolminfee"] tx_A = self.wallet.send_self_transfer( from_node=node, - fee=(mempoolmin_feerate / 1000) * (A_weight // 4) + Decimal('0.000001'), + fee_rate=mempoolmin_feerate, target_weight=A_weight, utxo_to_spend=rbf_utxo, confirmed_only=True @@ -77,7 +77,7 @@ class MempoolLimitTest(BitcoinTestFramework): non_cpfp_carveout_weight = 40001 # EXTRA_DESCENDANT_TX_SIZE_LIMIT + 1 tx_C = self.wallet.create_self_transfer( target_weight=non_cpfp_carveout_weight, - fee = (mempoolmin_feerate / 1000) * (non_cpfp_carveout_weight // 4) + Decimal('0.000001'), + fee_rate=mempoolmin_feerate, utxo_to_spend=tx_B["new_utxo"], confirmed_only=True ) @@ -109,7 +109,7 @@ class MempoolLimitTest(BitcoinTestFramework): # happen in the middle of package evaluation, as it can invalidate the coins cache. mempool_evicted_tx = self.wallet.send_self_transfer( from_node=node, - fee=(mempoolmin_feerate / 1000) * (evicted_weight // 4) + Decimal('0.000001'), + fee_rate=mempoolmin_feerate, target_weight=evicted_weight, confirmed_only=True ) @@ -135,11 +135,11 @@ class MempoolLimitTest(BitcoinTestFramework): parent_weight = 100000 num_big_parents = 3 assert_greater_than(parent_weight * num_big_parents, current_info["maxmempool"] - current_info["bytes"]) - parent_fee = (100 * mempoolmin_feerate / 1000) * (parent_weight // 4) + parent_feerate = 100 * mempoolmin_feerate big_parent_txids = [] for i in range(num_big_parents): - parent = self.wallet.create_self_transfer(fee=parent_fee, target_weight=parent_weight, confirmed_only=True) + parent = self.wallet.create_self_transfer(fee_rate=parent_feerate, target_weight=parent_weight, confirmed_only=True) parent_utxos.append(parent["new_utxo"]) package_hex.append(parent["hex"]) big_parent_txids.append(parent["txid"]) @@ -314,18 +314,20 @@ class MempoolLimitTest(BitcoinTestFramework): target_weight_each = 200000 assert_greater_than(target_weight_each * 2, node.getmempoolinfo()["maxmempool"] - node.getmempoolinfo()["bytes"]) # Should be a true CPFP: parent's feerate is just below mempool min feerate - parent_fee = (mempoolmin_feerate / 1000) * (target_weight_each // 4) - Decimal("0.00001") + parent_feerate = mempoolmin_feerate - Decimal("0.000001") # 0.1 sats/vbyte below min feerate # Parent + child is above mempool minimum feerate - child_fee = (worst_feerate_btcvb) * (target_weight_each // 4) - Decimal("0.00001") + child_feerate = (worst_feerate_btcvb * 1000) - Decimal("0.000001") # 0.1 sats/vbyte below worst feerate # However, when eviction is triggered, these transactions should be at the bottom. # This assertion assumes parent and child are the same size. miniwallet.rescan_utxos() - tx_parent_just_below = miniwallet.create_self_transfer(fee=parent_fee, target_weight=target_weight_each) - tx_child_just_above = miniwallet.create_self_transfer(utxo_to_spend=tx_parent_just_below["new_utxo"], fee=child_fee, target_weight=target_weight_each) + tx_parent_just_below = miniwallet.create_self_transfer(fee_rate=parent_feerate, target_weight=target_weight_each) + tx_child_just_above = miniwallet.create_self_transfer(utxo_to_spend=tx_parent_just_below["new_utxo"], fee_rate=child_feerate, target_weight=target_weight_each) # This package ranks below the lowest descendant package in the mempool - assert_greater_than(worst_feerate_btcvb, (parent_fee + child_fee) / (tx_parent_just_below["tx"].get_vsize() + tx_child_just_above["tx"].get_vsize())) - assert_greater_than(mempoolmin_feerate, (parent_fee) / (tx_parent_just_below["tx"].get_vsize())) - assert_greater_than((parent_fee + child_fee) / (tx_parent_just_below["tx"].get_vsize() + tx_child_just_above["tx"].get_vsize()), mempoolmin_feerate / 1000) + package_fee = tx_parent_just_below["fee"] + tx_child_just_above["fee"] + package_vsize = tx_parent_just_below["tx"].get_vsize() + tx_child_just_above["tx"].get_vsize() + assert_greater_than(worst_feerate_btcvb, package_fee / package_vsize) + assert_greater_than(mempoolmin_feerate, tx_parent_just_below["fee"] / (tx_parent_just_below["tx"].get_vsize())) + assert_greater_than(package_fee / package_vsize, mempoolmin_feerate / 1000) res = node.submitpackage([tx_parent_just_below["hex"], tx_child_just_above["hex"]]) for wtxid in [tx_parent_just_below["wtxid"], tx_child_just_above["wtxid"]]: assert_equal(res["tx-results"][wtxid]["error"], "mempool full") diff --git a/test/functional/mempool_package_onemore.py b/test/functional/mempool_package_onemore.py index 98b397e32b..632425814a 100755 --- a/test/functional/mempool_package_onemore.py +++ b/test/functional/mempool_package_onemore.py @@ -40,28 +40,37 @@ class MempoolPackagesTest(BitcoinTestFramework): for _ in range(DEFAULT_ANCESTOR_LIMIT - 4): utxo, = self.chain_tx([utxo]) chain.append(utxo) - second_chain, = self.chain_tx([self.wallet.get_utxo()]) + second_chain, = self.chain_tx([self.wallet.get_utxo(confirmed_only=True)]) # Check mempool has DEFAULT_ANCESTOR_LIMIT + 1 transactions in it assert_equal(len(self.nodes[0].getrawmempool()), DEFAULT_ANCESTOR_LIMIT + 1) # Adding one more transaction on to the chain should fail. assert_raises_rpc_error(-26, "too-long-mempool-chain, too many unconfirmed ancestors [limit: 25]", self.chain_tx, [utxo]) - # ...even if it chains on from some point in the middle of the chain. + # ... or if it chains on from some point in the middle of the chain. assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[2]]) assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[1]]) # ...even if it chains on to two parent transactions with one in the chain. assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[0], second_chain]) # ...especially if its > 40k weight assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[0]], num_outputs=350) + # ...even if it's submitted with other transactions + replaceable_tx = self.wallet.create_self_transfer_multi(utxos_to_spend=[chain[0]]) + txns = [replaceable_tx["tx"], self.wallet.create_self_transfer_multi(utxos_to_spend=replaceable_tx["new_utxos"])["tx"]] + txns_hex = [tx.serialize().hex() for tx in txns] + assert_equal(self.nodes[0].testmempoolaccept(txns_hex)[0]["reject-reason"], "too-long-mempool-chain") + pkg_result = self.nodes[0].submitpackage(txns_hex) + assert "too-long-mempool-chain" in pkg_result["tx-results"][txns[0].getwtxid()]["error"] + assert_equal(pkg_result["tx-results"][txns[1].getwtxid()]["error"], "bad-txns-inputs-missingorspent") # But not if it chains directly off the first transaction - replacable_tx = self.wallet.send_self_transfer_multi(from_node=self.nodes[0], utxos_to_spend=[chain[0]])['tx'] + self.nodes[0].sendrawtransaction(replaceable_tx["hex"]) # and the second chain should work just fine self.chain_tx([second_chain]) - # Make sure we can RBF the chain which used our carve-out rule - replacable_tx.vout[0].nValue -= 1000000 - self.nodes[0].sendrawtransaction(replacable_tx.serialize().hex()) + # Ensure an individual transaction with single direct conflict can RBF the chain which used our carve-out rule + replacement_tx = replaceable_tx["tx"] + replacement_tx.vout[0].nValue -= 1000000 + self.nodes[0].sendrawtransaction(replacement_tx.serialize().hex()) # Finally, check that we added two transactions assert_equal(len(self.nodes[0].getrawmempool()), DEFAULT_ANCESTOR_LIMIT + 3) diff --git a/test/functional/mempool_package_rbf.py b/test/functional/mempool_package_rbf.py new file mode 100755 index 0000000000..ceb9530394 --- /dev/null +++ b/test/functional/mempool_package_rbf.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from decimal import Decimal + +from test_framework.messages import ( + COIN, + MAX_BIP125_RBF_SEQUENCE, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.mempool_util import fill_mempool +from test_framework.util import ( + assert_greater_than_or_equal, + assert_equal, +) +from test_framework.wallet import ( + DEFAULT_FEE, + MiniWallet, +) + +MAX_REPLACEMENT_CANDIDATES = 100 + +# Value high enough to cause evictions in each subtest +# for typical cases +DEFAULT_CHILD_FEE = DEFAULT_FEE * 4 + +class PackageRBFTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.setup_clean_chain = True + # Required for fill_mempool() + self.extra_args = [[ + "-datacarriersize=100000", + "-maxmempool=5", + ]] * self.num_nodes + + def assert_mempool_contents(self, expected=None): + """Assert that all transactions in expected are in the mempool, + and no additional ones exist. + """ + if not expected: + expected = [] + mempool = self.nodes[0].getrawmempool(verbose=False) + assert_equal(len(mempool), len(expected)) + for tx in expected: + assert tx.rehash() in mempool + + def create_simple_package(self, parent_coin, parent_fee=DEFAULT_FEE, child_fee=DEFAULT_CHILD_FEE, heavy_child=False): + """Create a 1 parent 1 child package using the coin passed in as the parent's input. The + parent has 1 output, used to fund 1 child transaction. + All transactions signal BIP125 replaceability, but nSequence changes based on self.ctr. This + prevents identical txids between packages when the parents spend the same coin and have the + same fee (i.e. 0sat). + + returns tuple (hex serialized txns, CTransaction objects) + """ + self.ctr += 1 + # Use fee_rate=0 because create_self_transfer will use the default fee_rate value otherwise. + # Passing in fee>0 overrides fee_rate, so this still works for non-zero parent_fee. + parent_result = self.wallet.create_self_transfer( + fee=parent_fee, + utxo_to_spend=parent_coin, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + num_child_outputs = 10 if heavy_child else 1 + child_result = self.wallet.create_self_transfer_multi( + utxos_to_spend=[parent_result["new_utxo"]], + num_outputs=num_child_outputs, + fee_per_output=int(child_fee * COIN // num_child_outputs), + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + package_hex = [parent_result["hex"], child_result["hex"]] + package_txns = [parent_result["tx"], child_result["tx"]] + return package_hex, package_txns + + def run_test(self): + # Counter used to count the number of times we constructed packages. Since we're constructing parent transactions with the same + # coins (to create conflicts), and perhaps giving them the same fee, we might accidentally just create the same transaction again. + # To prevent this, set nSequences to MAX_BIP125_RBF_SEQUENCE - self.ctr. + self.ctr = 0 + + self.log.info("Generate blocks to create UTXOs") + self.wallet = MiniWallet(self.nodes[0]) + + # Make more than enough coins for the sum of all tests, + # otherwise a wallet rescan is needed later + self.generate(self.wallet, 300) + self.coins = self.wallet.get_utxos(mark_as_spent=False) + + self.test_package_rbf_basic() + self.test_package_rbf_singleton() + self.test_package_rbf_additional_fees() + self.test_package_rbf_max_conflicts() + self.test_too_numerous_ancestors() + self.test_package_rbf_with_wrong_pkg_size() + self.test_insufficient_feerate() + self.test_wrong_conflict_cluster_size_linear() + self.test_wrong_conflict_cluster_size_parents_child() + self.test_wrong_conflict_cluster_size_parent_children() + self.test_0fee_package_rbf() + self.test_child_conflicts_parent_mempool_ancestor() + + def test_package_rbf_basic(self): + self.log.info("Test that a child can pay to replace its parents' conflicts of cluster size 2") + node = self.nodes[0] + # Reuse the same coins so that the transactions conflict with one another. + parent_coin = self.coins.pop() + package_hex1, package_txns1 = self.create_simple_package(parent_coin, DEFAULT_FEE, DEFAULT_FEE) + package_hex2, package_txns2 = self.create_simple_package(parent_coin, DEFAULT_FEE, DEFAULT_CHILD_FEE) + node.submitpackage(package_hex1) + self.assert_mempool_contents(expected=package_txns1) + + # Make sure 2nd node gets set up for basic package RBF + self.sync_all() + + # Test run rejected because conflicts are not allowed in subpackage evaluation + testres = node.testmempoolaccept(package_hex2) + assert_equal(testres[0]["reject-reason"], "bip125-replacement-disallowed") + + # But accepted during normal submission + submitres = node.submitpackage(package_hex2) + assert_equal(set(submitres["replaced-transactions"]), set([tx.rehash() for tx in package_txns1])) + self.assert_mempool_contents(expected=package_txns2) + + # Make sure 2nd node gets a basic package RBF over p2p + self.sync_all() + + self.generate(node, 1) + + def test_package_rbf_singleton(self): + self.log.info("Test child can pay to replace a parent's single conflicted tx") + node = self.nodes[0] + + # Make singleton tx to conflict with in next batch + singleton_coin = self.coins.pop() + singleton_tx = self.wallet.create_self_transfer(utxo_to_spend=singleton_coin) + node.sendrawtransaction(singleton_tx["hex"]) + self.assert_mempool_contents(expected=[singleton_tx["tx"]]) + + package_hex, package_txns = self.create_simple_package(singleton_coin, DEFAULT_FEE, singleton_tx["fee"] * 2) + + submitres = node.submitpackage(package_hex) + assert_equal(submitres["replaced-transactions"], [singleton_tx["tx"].rehash()]) + self.assert_mempool_contents(expected=package_txns) + + self.generate(node, 1) + + def test_package_rbf_additional_fees(self): + self.log.info("Check Package RBF must increase the absolute fee") + node = self.nodes[0] + coin = self.coins.pop() + + package_hex1, package_txns1 = self.create_simple_package(coin, parent_fee=DEFAULT_FEE, child_fee=DEFAULT_CHILD_FEE, heavy_child=True) + assert_greater_than_or_equal(1000, package_txns1[-1].get_vsize()) + node.submitpackage(package_hex1) + self.assert_mempool_contents(expected=package_txns1) + + PACKAGE_FEE = DEFAULT_FEE + DEFAULT_CHILD_FEE + PACKAGE_FEE_MINUS_ONE = PACKAGE_FEE - Decimal("0.00000001") + + # Package 2 has a higher feerate but lower absolute fee + package_hex2, package_txns2 = self.create_simple_package(coin, parent_fee=DEFAULT_FEE, child_fee=DEFAULT_CHILD_FEE - Decimal("0.00000001")) + pkg_results2 = node.submitpackage(package_hex2) + assert_equal(f"package RBF failed: insufficient anti-DoS fees, rejecting replacement {package_txns2[1].rehash()}, less fees than conflicting txs; {PACKAGE_FEE_MINUS_ONE} < {PACKAGE_FEE}", pkg_results2["package_msg"]) + self.assert_mempool_contents(expected=package_txns1) + + self.log.info("Check replacement pays for incremental bandwidth") + package_hex3, package_txns3 = self.create_simple_package(coin, parent_fee=DEFAULT_FEE, child_fee=DEFAULT_CHILD_FEE) + pkg_results3 = node.submitpackage(package_hex3) + assert_equal(f"package RBF failed: insufficient anti-DoS fees, rejecting replacement {package_txns3[1].rehash()}, not enough additional fees to relay; 0.00 < 0.00000{sum([tx.get_vsize() for tx in package_txns3])}", pkg_results3["package_msg"]) + + self.assert_mempool_contents(expected=package_txns1) + self.generate(node, 1) + + self.log.info("Check Package RBF must have strict cpfp structure") + coin = self.coins.pop() + package_hex4, package_txns4 = self.create_simple_package(coin, parent_fee=DEFAULT_FEE, child_fee=DEFAULT_CHILD_FEE) + node.submitpackage(package_hex4) + self.assert_mempool_contents(expected=package_txns4) + package_hex5, package_txns5 = self.create_simple_package(coin, parent_fee=DEFAULT_CHILD_FEE, child_fee=DEFAULT_CHILD_FEE - Decimal("0.00000001")) + pkg_results5 = node.submitpackage(package_hex5) + assert 'package RBF failed: package feerate is less than parent feerate' in pkg_results5["package_msg"] + + self.assert_mempool_contents(expected=package_txns4) + self.generate(node, 1) + + def test_package_rbf_max_conflicts(self): + node = self.nodes[0] + self.log.info("Check Package RBF cannot replace more than MAX_REPLACEMENT_CANDIDATES transactions") + num_coins = 51 + parent_coins = self.coins[:num_coins] + del self.coins[:num_coins] + + # Original transactions: 51 transactions with 1 descendants each -> 102 total transactions + size_two_clusters = [] + for coin in parent_coins: + size_two_clusters.append(self.wallet.send_self_transfer_chain(from_node=node, chain_length=2, utxo_to_spend=coin)) + expected_txns = [txn["tx"] for parent_child_txns in size_two_clusters for txn in parent_child_txns] + assert_equal(len(expected_txns), num_coins * 2) + self.assert_mempool_contents(expected=expected_txns) + + # parent feeerate needs to be high enough for minrelay + # child feerate needs to be large enough to trigger package rbf with a very large parent and + # pay for all evicted fees. maxfeerate turned off for all submissions since child feerate + # is extremely high + parent_fee_per_conflict = 10000 + child_feerate = 10000 * DEFAULT_FEE + + # Conflict against all transactions by double-spending each parent, causing 102 evictions + package_parent = self.wallet.create_self_transfer_multi(utxos_to_spend=parent_coins, fee_per_output=parent_fee_per_conflict) + package_child = self.wallet.create_self_transfer(fee_rate=child_feerate, utxo_to_spend=package_parent["new_utxos"][0]) + + pkg_results = node.submitpackage([package_parent["hex"], package_child["hex"]], maxfeerate=0) + assert_equal(f"package RBF failed: too many potential replacements, rejecting replacement {package_child['tx'].rehash()}; too many potential replacements (102 > 100)\n", pkg_results["package_msg"]) + self.assert_mempool_contents(expected=expected_txns) + + # Make singleton tx to conflict with in next batch + singleton_coin = self.coins.pop() + singleton_tx = self.wallet.create_self_transfer(utxo_to_spend=singleton_coin) + node.sendrawtransaction(singleton_tx["hex"]) + expected_txns.append(singleton_tx["tx"]) + + # Double-spend same set minus last, and double-spend singleton. This hits 101 evictions; should still fail. + # N.B. we can't RBF just a child tx in the clusters, as that would make resulting cluster of size 3. + double_spending_coins = parent_coins[:-1] + [singleton_coin] + package_parent = self.wallet.create_self_transfer_multi(utxos_to_spend=double_spending_coins, fee_per_output=parent_fee_per_conflict) + package_child = self.wallet.create_self_transfer(fee_rate=child_feerate, utxo_to_spend=package_parent["new_utxos"][0]) + pkg_results = node.submitpackage([package_parent["hex"], package_child["hex"]], maxfeerate=0) + assert_equal(f"package RBF failed: too many potential replacements, rejecting replacement {package_child['tx'].rehash()}; too many potential replacements (101 > 100)\n", pkg_results["package_msg"]) + self.assert_mempool_contents(expected=expected_txns) + + # Finally, evict MAX_REPLACEMENT_CANDIDATES + package_parent = self.wallet.create_self_transfer_multi(utxos_to_spend=parent_coins[:-1], fee_per_output=parent_fee_per_conflict) + package_child = self.wallet.create_self_transfer(fee_rate=child_feerate, utxo_to_spend=package_parent["new_utxos"][0]) + pkg_results = node.submitpackage([package_parent["hex"], package_child["hex"]], maxfeerate=0) + assert_equal(pkg_results["package_msg"], "success") + self.assert_mempool_contents(expected=[singleton_tx["tx"], size_two_clusters[-1][0]["tx"], size_two_clusters[-1][1]["tx"], package_parent["tx"], package_child["tx"]] ) + + self.generate(node, 1) + + def test_too_numerous_ancestors(self): + self.log.info("Test that package RBF doesn't work with packages larger than 2 due to ancestors") + node = self.nodes[0] + coin = self.coins.pop() + + package_hex1, package_txns1 = self.create_simple_package(coin, DEFAULT_FEE, DEFAULT_CHILD_FEE) + node.submitpackage(package_hex1) + self.assert_mempool_contents(expected=package_txns1) + + # Double-spends the original package + self.ctr += 1 + parent_result1 = self.wallet.create_self_transfer( + fee=DEFAULT_FEE, + utxo_to_spend=coin, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + coin2 = self.coins.pop() + + # Added to make package too large for package RBF; + # it will enter mempool individually + self.ctr += 1 + parent_result2 = self.wallet.create_self_transfer( + fee=DEFAULT_FEE, + utxo_to_spend=coin2, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + # Child that spends both, violating cluster size rule due + # to in-mempool ancestry + self.ctr += 1 + child_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_CHILD_FEE * COIN), + utxos_to_spend=[parent_result1["new_utxo"], parent_result2["new_utxo"]], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + package_hex2 = [parent_result1["hex"], parent_result2["hex"], child_result["hex"]] + package_txns2_succeed = [parent_result2["tx"]] + + pkg_result = node.submitpackage(package_hex2) + assert_equal(pkg_result["package_msg"], 'package RBF failed: new transaction cannot have mempool ancestors') + self.assert_mempool_contents(expected=package_txns1 + package_txns2_succeed) + self.generate(node, 1) + + def test_wrong_conflict_cluster_size_linear(self): + self.log.info("Test that conflicting with a cluster not sized two is rejected: linear chain") + node = self.nodes[0] + + # Coins we will conflict with + coin1 = self.coins.pop() + coin2 = self.coins.pop() + coin3 = self.coins.pop() + + # Three transactions chained; package RBF against any of these + # should be rejected + self.ctr += 1 + parent_result = self.wallet.create_self_transfer( + fee=DEFAULT_FEE, + utxo_to_spend=coin1, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + self.ctr += 1 + child_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * COIN), + utxos_to_spend=[parent_result["new_utxo"], coin2], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + self.ctr += 1 + grandchild_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * COIN), + utxos_to_spend=[child_result["new_utxos"][0], coin3], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + expected_txns = [parent_result["tx"], child_result["tx"], grandchild_result["tx"]] + for tx in expected_txns: + node.sendrawtransaction(tx.serialize().hex()) + self.assert_mempool_contents(expected=expected_txns) + + # Now make conflicting packages for each coin + package_hex1, package_txns1 = self.create_simple_package(coin1, DEFAULT_FEE, DEFAULT_CHILD_FEE) + + package_result = node.submitpackage(package_hex1) + assert_equal(f"package RBF failed: {parent_result['tx'].rehash()} has 2 descendants, max 1 allowed", package_result["package_msg"]) + + package_hex2, package_txns2 = self.create_simple_package(coin2, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex2) + assert_equal(f"package RBF failed: {child_result['tx'].rehash()} has both ancestor and descendant, exceeding cluster limit of 2", package_result["package_msg"]) + + package_hex3, package_txns3 = self.create_simple_package(coin3, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex3) + assert_equal(f"package RBF failed: {grandchild_result['tx'].rehash()} has 2 ancestors, max 1 allowed", package_result["package_msg"]) + + # Check that replacements were actually rejected + self.assert_mempool_contents(expected=expected_txns) + self.generate(node, 1) + + def test_wrong_conflict_cluster_size_parents_child(self): + self.log.info("Test that conflicting with a cluster not sized two is rejected: two parents one child") + node = self.nodes[0] + + # Coins we will conflict with + coin1 = self.coins.pop() + coin2 = self.coins.pop() + coin3 = self.coins.pop() + + self.ctr += 1 + parent1_result = self.wallet.create_self_transfer( + fee=DEFAULT_FEE, + utxo_to_spend=coin1, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + self.ctr += 1 + parent2_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * COIN), + utxos_to_spend=[coin2], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + self.ctr += 1 + child_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * COIN), + utxos_to_spend=[parent1_result["new_utxo"], parent2_result["new_utxos"][0], coin3], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + expected_txns = [parent1_result["tx"], parent2_result["tx"], child_result["tx"]] + for tx in expected_txns: + node.sendrawtransaction(tx.serialize().hex()) + self.assert_mempool_contents(expected=expected_txns) + + # Now make conflicting packages for each coin + package_hex1, package_txns1 = self.create_simple_package(coin1, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex1) + assert_equal(f"package RBF failed: {parent1_result['tx'].rehash()} is not the only parent of child {child_result['tx'].rehash()}", package_result["package_msg"]) + + package_hex2, package_txns2 = self.create_simple_package(coin2, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex2) + assert_equal(f"package RBF failed: {parent2_result['tx'].rehash()} is not the only parent of child {child_result['tx'].rehash()}", package_result["package_msg"]) + + package_hex3, package_txns3 = self.create_simple_package(coin3, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex3) + assert_equal(f"package RBF failed: {child_result['tx'].rehash()} has 2 ancestors, max 1 allowed", package_result["package_msg"]) + + # Check that replacements were actually rejected + self.assert_mempool_contents(expected=expected_txns) + self.generate(node, 1) + + def test_wrong_conflict_cluster_size_parent_children(self): + self.log.info("Test that conflicting with a cluster not sized two is rejected: one parent two children") + node = self.nodes[0] + + # Coins we will conflict with + coin1 = self.coins.pop() + coin2 = self.coins.pop() + coin3 = self.coins.pop() + + self.ctr += 1 + parent_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * COIN), + num_outputs=2, + utxos_to_spend=[coin1], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + self.ctr += 1 + child1_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * COIN), + utxos_to_spend=[parent_result["new_utxos"][0], coin2], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + self.ctr += 1 + child2_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * COIN), + utxos_to_spend=[parent_result["new_utxos"][1], coin3], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + # Submit them to mempool + expected_txns = [parent_result["tx"], child1_result["tx"], child2_result["tx"]] + for tx in expected_txns: + node.sendrawtransaction(tx.serialize().hex()) + self.assert_mempool_contents(expected=expected_txns) + + # Now make conflicting packages for each coin + package_hex1, package_txns1 = self.create_simple_package(coin1, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex1) + assert_equal(f"package RBF failed: {parent_result['tx'].rehash()} has 2 descendants, max 1 allowed", package_result["package_msg"]) + + package_hex2, package_txns2 = self.create_simple_package(coin2, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex2) + assert_equal(f"package RBF failed: {child1_result['tx'].rehash()} is not the only child of parent {parent_result['tx'].rehash()}", package_result["package_msg"]) + + package_hex3, package_txns3 = self.create_simple_package(coin3, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex3) + assert_equal(f"package RBF failed: {child2_result['tx'].rehash()} is not the only child of parent {parent_result['tx'].rehash()}", package_result["package_msg"]) + + # Check that replacements were actually rejected + self.assert_mempool_contents(expected=expected_txns) + self.generate(node, 1) + + def test_package_rbf_with_wrong_pkg_size(self): + self.log.info("Test that package RBF doesn't work with packages larger than 2 due to pkg size") + node = self.nodes[0] + coin1 = self.coins.pop() + coin2 = self.coins.pop() + + # Two packages to require multiple direct conflicts, easier to set up illicit pkg size + package_hex1, package_txns1 = self.create_simple_package(coin1, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_hex2, package_txns2 = self.create_simple_package(coin2, DEFAULT_FEE, DEFAULT_CHILD_FEE) + + node.submitpackage(package_hex1) + node.submitpackage(package_hex2) + + self.assert_mempool_contents(expected=package_txns1 + package_txns2) + assert_equal(len(node.getrawmempool()), 4) + + # Double-spends the first package + self.ctr += 1 + parent_result1 = self.wallet.create_self_transfer( + fee=DEFAULT_FEE, + utxo_to_spend=coin1, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + # Double-spends the second package + self.ctr += 1 + parent_result2 = self.wallet.create_self_transfer( + fee=DEFAULT_FEE, + utxo_to_spend=coin2, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + # Child that spends both, violating cluster size rule due + # to pkg size + self.ctr += 1 + child_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_CHILD_FEE * COIN), + utxos_to_spend=[parent_result1["new_utxo"], parent_result2["new_utxo"]], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + package_hex3 = [parent_result1["hex"], parent_result2["hex"], child_result["hex"]] + + pkg_result = node.submitpackage(package_hex3) + assert_equal(pkg_result["package_msg"], 'package RBF failed: package must be 1-parent-1-child') + self.assert_mempool_contents(expected=package_txns1 + package_txns2) + self.generate(node, 1) + + def test_insufficient_feerate(self): + self.log.info("Check Package RBF must beat feerate of direct conflict") + node = self.nodes[0] + coin = self.coins.pop() + + # Non-cpfp structure + package_hex1, package_txns1 = self.create_simple_package(coin, parent_fee=DEFAULT_CHILD_FEE, child_fee=DEFAULT_FEE) + node.submitpackage(package_hex1) + self.assert_mempool_contents(expected=package_txns1) + + # Package 2 feerate is below the feerate of directly conflicted parent, so it fails even though + # total fees are higher than the original package + package_hex2, package_txns2 = self.create_simple_package(coin, parent_fee=DEFAULT_CHILD_FEE - Decimal("0.00000001"), child_fee=DEFAULT_CHILD_FEE) + pkg_results2 = node.submitpackage(package_hex2) + assert_equal(pkg_results2["package_msg"], 'package RBF failed: insufficient feerate: does not improve feerate diagram') + self.assert_mempool_contents(expected=package_txns1) + self.generate(node, 1) + + def test_0fee_package_rbf(self): + self.log.info("Test package RBF: TRUC 0-fee parent + high-fee child replaces parent's conflicts") + node = self.nodes[0] + # Reuse the same coins so that the transactions conflict with one another. + self.wallet.rescan_utxos() + parent_coin = self.wallet.get_utxo(confirmed_only=True) + + # package1 pays default fee on both transactions + parent1 = self.wallet.create_self_transfer(utxo_to_spend=parent_coin, version=3) + child1 = self.wallet.create_self_transfer(utxo_to_spend=parent1["new_utxo"], version=3) + package_hex1 = [parent1["hex"], child1["hex"]] + fees_package1 = parent1["fee"] + child1["fee"] + submitres1 = node.submitpackage(package_hex1) + assert_equal(submitres1["package_msg"], "success") + self.assert_mempool_contents([parent1["tx"], child1["tx"]]) + + # package2 has a 0-fee parent (conflicting with package1) and very high fee child + parent2 = self.wallet.create_self_transfer(utxo_to_spend=parent_coin, fee=0, fee_rate=0, version=3) + child2 = self.wallet.create_self_transfer(utxo_to_spend=parent2["new_utxo"], fee=fees_package1*10, version=3) + package_hex2 = [parent2["hex"], child2["hex"]] + + submitres2 = node.submitpackage(package_hex2) + assert_equal(submitres2["package_msg"], "success") + assert_equal(set(submitres2["replaced-transactions"]), set([parent1["txid"], child1["txid"]])) + self.assert_mempool_contents([parent2["tx"], child2["tx"]]) + + self.generate(node, 1) + + def test_child_conflicts_parent_mempool_ancestor(self): + fill_mempool(self, self.nodes[0]) + # Reset coins since we filled the mempool with current coins + self.coins = self.wallet.get_utxos(mark_as_spent=False, confirmed_only=True) + + self.log.info("Test that package RBF doesn't have issues with mempool<->package conflicts via inconsistency") + node = self.nodes[0] + coin = self.coins.pop() + + self.ctr += 1 + grandparent_result = self.wallet.create_self_transfer( + fee=DEFAULT_FEE, + utxo_to_spend=coin, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + node.sendrawtransaction(grandparent_result["hex"]) + + # Now make package of two descendants that looks + # like a cpfp where the parent can't get in on its own + self.ctr += 1 + parent_result = self.wallet.create_self_transfer( + fee_rate=Decimal('0.00001000'), + utxo_to_spend=grandparent_result["new_utxo"], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + # Last tx double-spends grandparent's coin, + # which is not inside the current package + self.ctr += 1 + child_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_CHILD_FEE * COIN), + utxos_to_spend=[parent_result["new_utxo"], coin], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + pkg_result = node.submitpackage([parent_result["hex"], child_result["hex"]]) + assert_equal(pkg_result["package_msg"], 'package RBF failed: new transaction cannot have mempool ancestors') + mempool_info = node.getrawmempool() + assert grandparent_result["txid"] in mempool_info + assert parent_result["txid"] not in mempool_info + assert child_result["txid"] not in mempool_info + +if __name__ == "__main__": + PackageRBFTest().main() diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index b23ec1028b..d10e47e036 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -142,7 +142,8 @@ class AddrTest(BitcoinTestFramework): msg = self.setup_addr_msg(1010) with self.nodes[0].assert_debug_log(['addr message size = 1010']): - addr_source.send_and_ping(msg) + addr_source.send_message(msg) + addr_source.wait_for_disconnect() self.nodes[0].disconnect_p2ps() diff --git a/test/functional/p2p_addrv2_relay.py b/test/functional/p2p_addrv2_relay.py index ea114e7d70..4ec8e0bc04 100755 --- a/test/functional/p2p_addrv2_relay.py +++ b/test/functional/p2p_addrv2_relay.py @@ -86,11 +86,6 @@ class AddrTest(BitcoinTestFramework): addr_source = self.nodes[0].add_p2p_connection(P2PInterface()) msg = msg_addrv2() - self.log.info('Send too-large addrv2 message') - msg.addrs = ADDRS * 101 - with self.nodes[0].assert_debug_log(['addrv2 message size = 1010']): - addr_source.send_and_ping(msg) - self.log.info('Check that addrv2 message content is relayed and added to addrman') addr_receiver = self.nodes[0].add_p2p_connection(AddrReceiver()) msg.addrs = ADDRS @@ -106,6 +101,13 @@ class AddrTest(BitcoinTestFramework): assert addr_receiver.addrv2_received_and_checked assert_equal(len(self.nodes[0].getnodeaddresses(count=0, network="i2p")), 0) + self.log.info('Send too-large addrv2 message') + msg.addrs = ADDRS * 101 + with self.nodes[0].assert_debug_log(['addrv2 message size = 1010']): + addr_source.send_message(msg) + addr_source.wait_for_disconnect() + + if __name__ == '__main__': AddrTest().main() diff --git a/test/functional/p2p_disconnect_ban.py b/test/functional/p2p_disconnect_ban.py index 678b006886..e47f9c732b 100755 --- a/test/functional/p2p_disconnect_ban.py +++ b/test/functional/p2p_disconnect_ban.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2022 The Bitcoin Core developers +# Copyright (c) 2014-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node disconnect and ban behavior""" @@ -18,7 +18,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.supports_cli = False def run_test(self): - self.log.info("Connect nodes both way") + self.log.info("Connect nodes both ways") # By default, the test framework sets up an addnode connection from # node 1 --> node0. By connecting node0 --> node 1, we're left with # the two nodes being connected both ways. @@ -84,7 +84,7 @@ class DisconnectBanTest(BitcoinTestFramework): assert_equal("192.168.0.1/32", listBeforeShutdown[2]['address']) self.log.info("setban: test banning with absolute timestamp") - self.nodes[1].setban("192.168.0.2", "add", old_time + 120, True) + self.nodes[1].setban("192.168.0.2", "add", old_time + 120, absolute=True) # Move time forward by 3 seconds so the fourth ban has expired self.nodes[1].setmocktime(old_time + 3) @@ -102,7 +102,9 @@ class DisconnectBanTest(BitcoinTestFramework): assert_equal(ban["ban_duration"], 120) assert_equal(ban["time_remaining"], 117) - self.restart_node(1) + # Keep mocktime, to avoid ban expiry when restart takes longer than + # time_remaining + self.restart_node(1, extra_args=[f"-mocktime={old_time+4}"]) listAfterShutdown = self.nodes[1].listbanned() assert_equal("127.0.0.0/24", listAfterShutdown[0]['address']) @@ -113,7 +115,7 @@ class DisconnectBanTest(BitcoinTestFramework): # Clear ban lists self.nodes[1].clearbanned() - self.log.info("Connect nodes both way") + self.log.info("Connect nodes both ways") self.connect_nodes(0, 1) self.connect_nodes(1, 0) diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 40a69936bc..8e459ba676 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -5,7 +5,6 @@ """Test node responses to invalid network messages.""" import random -import struct import time from test_framework.messages import ( @@ -233,7 +232,7 @@ class InvalidMessagesTest(BitcoinTestFramework): '208d')) # port def test_addrv2_unrecognized_network(self): - now_hex = struct.pack('<I', int(time.time())).hex() + now_hex = int(time.time()).to_bytes(4, "little").hex() self.test_addrv2('unrecognized network', [ 'received: addrv2 (25 bytes)', @@ -261,7 +260,9 @@ class InvalidMessagesTest(BitcoinTestFramework): msg_type = msg.msgtype.decode('ascii') self.log.info("Test {} message of size {} is logged as misbehaving".format(msg_type, size)) with self.nodes[0].assert_debug_log(['Misbehaving', '{} message size = {}'.format(msg_type, size)]): - self.nodes[0].add_p2p_connection(P2PInterface()).send_and_ping(msg) + conn = self.nodes[0].add_p2p_connection(P2PInterface()) + conn.send_message(msg) + conn.wait_for_disconnect() self.nodes[0].disconnect_p2ps() def test_oversized_inv_msg(self): @@ -322,7 +323,8 @@ class InvalidMessagesTest(BitcoinTestFramework): # delete arbitrary block header somewhere in the middle to break link del block_headers[random.randrange(1, len(block_headers)-1)] with self.nodes[0].assert_debug_log(expected_msgs=MISBEHAVING_NONCONTINUOUS_HEADERS_MSGS): - peer.send_and_ping(msg_headers(block_headers)) + peer.send_message(msg_headers(block_headers)) + peer.wait_for_disconnect() self.nodes[0].disconnect_p2ps() def test_resource_exhaustion(self): diff --git a/test/functional/p2p_mutated_blocks.py b/test/functional/p2p_mutated_blocks.py index 737edaf5bf..708b19b1e5 100755 --- a/test/functional/p2p_mutated_blocks.py +++ b/test/functional/p2p_mutated_blocks.py @@ -55,7 +55,7 @@ class MutatedBlocksTest(BitcoinTestFramework): # Create mutated version of the block by changing the transaction # version on the self-transfer. mutated_block = copy.deepcopy(block) - mutated_block.vtx[1].nVersion = 4 + mutated_block.vtx[1].version = 4 # Announce the new block via a compact block through the honest relayer cmpctblock = HeaderAndShortIDs() @@ -104,11 +104,10 @@ class MutatedBlocksTest(BitcoinTestFramework): block_missing_prev.hashPrevBlock = 123 block_missing_prev.solve() - # Attacker gets a DoS score of 10, not immediately disconnected, so we do it 10 times to get to 100 - for _ in range(10): - assert_equal(len(self.nodes[0].getpeerinfo()), 2) - with self.nodes[0].assert_debug_log(expected_msgs=["AcceptBlock FAILED (prev-blk-not-found)"]): - attacker.send_message(msg_block(block_missing_prev)) + # Check that non-connecting block causes disconnect + assert_equal(len(self.nodes[0].getpeerinfo()), 2) + with self.nodes[0].assert_debug_log(expected_msgs=["AcceptBlock FAILED (prev-blk-not-found)"]): + attacker.send_message(msg_block(block_missing_prev)) attacker.wait_for_disconnect(timeout=5) diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 45bbd7f1c3..d20cf41a72 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -5,7 +5,6 @@ """Test segwit transactions and blocks on P2P network.""" from decimal import Decimal import random -import struct import time from test_framework.blocktools import ( @@ -1165,16 +1164,16 @@ class SegWitTest(BitcoinTestFramework): if not self.wit.is_null(): flags |= 1 r = b"" - r += struct.pack("<i", self.nVersion) + r += self.version.to_bytes(4, "little") if flags: dummy = [] r += ser_vector(dummy) - r += struct.pack("<B", flags) + r += flags.to_bytes(1, "little") r += ser_vector(self.vin) r += ser_vector(self.vout) if flags & 1: r += self.wit.serialize() - r += struct.pack("<I", self.nLockTime) + r += self.nLockTime.to_bytes(4, "little") return r tx2 = BrokenCTransaction() @@ -1976,11 +1975,11 @@ class SegWitTest(BitcoinTestFramework): def serialize_with_bogus_witness(tx): flags = 3 r = b"" - r += struct.pack("<i", tx.nVersion) + r += tx.version.to_bytes(4, "little") if flags: dummy = [] r += ser_vector(dummy) - r += struct.pack("<B", flags) + r += flags.to_bytes(1, "little") r += ser_vector(tx.vin) r += ser_vector(tx.vout) if flags & 1: @@ -1990,7 +1989,7 @@ class SegWitTest(BitcoinTestFramework): for _ in range(len(tx.wit.vtxinwit), len(tx.vin)): tx.wit.vtxinwit.append(CTxInWitness()) r += tx.wit.serialize() - r += struct.pack("<I", tx.nLockTime) + r += tx.nLockTime.to_bytes(4, "little") return r class msg_bogus_tx(msg_tx): diff --git a/test/functional/p2p_sendheaders.py b/test/functional/p2p_sendheaders.py index 27a3aa8fb9..5c463267a1 100755 --- a/test/functional/p2p_sendheaders.py +++ b/test/functional/p2p_sendheaders.py @@ -71,19 +71,13 @@ f. Announce 1 more header that builds on that fork. Expect: no response. Part 5: Test handling of headers that don't connect. -a. Repeat 10 times: +a. Repeat 100 times: 1. Announce a header that doesn't connect. Expect: getheaders message 2. Send headers chain. Expect: getdata for the missing blocks, tip update. -b. Then send 9 more headers that don't connect. +b. Then send 99 more headers that don't connect. Expect: getheaders message each time. -c. Announce a header that does connect. - Expect: no response. -d. Announce 49 headers that don't connect. - Expect: getheaders message each time. -e. Announce one more that doesn't connect. - Expect: disconnect. """ from test_framework.blocktools import create_block, create_coinbase from test_framework.messages import CInv @@ -526,7 +520,8 @@ class SendHeadersTest(BitcoinTestFramework): # First we test that receipt of an unconnecting header doesn't prevent # chain sync. expected_hash = tip - for i in range(10): + NUM_HEADERS = 100 + for i in range(NUM_HEADERS): self.log.debug("Part 5.{}: starting...".format(i)) test_node.last_message.pop("getdata", None) blocks = [] @@ -550,41 +545,24 @@ class SendHeadersTest(BitcoinTestFramework): blocks = [] # Now we test that if we repeatedly don't send connecting headers, we # don't go into an infinite loop trying to get them to connect. - MAX_NUM_UNCONNECTING_HEADERS_MSGS = 10 - for _ in range(MAX_NUM_UNCONNECTING_HEADERS_MSGS + 1): + for _ in range(NUM_HEADERS + 1): blocks.append(create_block(tip, create_coinbase(height), block_time)) blocks[-1].solve() tip = blocks[-1].sha256 block_time += 1 height += 1 - for i in range(1, MAX_NUM_UNCONNECTING_HEADERS_MSGS): - # Send a header that doesn't connect, check that we get a getheaders. + for i in range(1, NUM_HEADERS): + with p2p_lock: + test_node.last_message.pop("getheaders", None) + # Send an empty header as a failed response to the received getheaders + # (from the previous iteration). Otherwise, the new headers will be + # treated as a response instead of as an announcement. + test_node.send_header_for_blocks([]) + # Send the actual unconnecting header, which should trigger a new getheaders. test_node.send_header_for_blocks([blocks[i]]) test_node.wait_for_getheaders(block_hash=expected_hash) - # Next header will connect, should re-set our count: - test_node.send_header_for_blocks([blocks[0]]) - expected_hash = blocks[0].sha256 - - # Remove the first two entries (blocks[1] would connect): - blocks = blocks[2:] - - # Now try to see how many unconnecting headers we can send - # before we get disconnected. Should be 5*MAX_NUM_UNCONNECTING_HEADERS_MSGS - for i in range(5 * MAX_NUM_UNCONNECTING_HEADERS_MSGS - 1): - # Send a header that doesn't connect, check that we get a getheaders. - test_node.send_header_for_blocks([blocks[i % len(blocks)]]) - test_node.wait_for_getheaders(block_hash=expected_hash) - - # Eventually this stops working. - test_node.send_header_for_blocks([blocks[-1]]) - - # Should get disconnected - test_node.wait_for_disconnect() - - self.log.info("Part 5: success!") - # Finally, check that the inv node never received a getdata request, # throughout the test assert "getdata" not in inv_node.last_message diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py index f368434895..776eaf5255 100755 --- a/test/functional/p2p_unrequested_blocks.py +++ b/test/functional/p2p_unrequested_blocks.py @@ -170,9 +170,11 @@ class AcceptBlockTest(BitcoinTestFramework): tip = next_block # Now send the block at height 5 and check that it wasn't accepted (missing header) - test_node.send_and_ping(msg_block(all_blocks[1])) + test_node.send_message(msg_block(all_blocks[1])) + test_node.wait_for_disconnect() assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getblock, all_blocks[1].hash) assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getblockheader, all_blocks[1].hash) + test_node = self.nodes[0].add_p2p_connection(P2PInterface()) # The block at height 5 should be accepted if we provide the missing header, though headers_message = msg_headers() diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index 65d7b4c422..37656341d2 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -9,10 +9,10 @@ import json import os from test_framework.address import address_to_scriptpubkey -from test_framework.blocktools import COINBASE_MATURITY -from test_framework.authproxy import JSONRPCException from test_framework.descriptors import descsum_create, drop_origins from test_framework.key import ECPubKey +from test_framework.messages import COIN +from test_framework.script_util import keys_to_multisig_script from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_raises_rpc_error, @@ -32,92 +32,54 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 3 self.supports_cli = False + self.enable_wallet_if_possible() - def get_keys(self): + def create_keys(self, num_keys): self.pub = [] self.priv = [] - node0, node1, node2 = self.nodes - for _ in range(self.nkeys): + for _ in range(num_keys): privkey, pubkey = generate_keypair(wif=True) self.pub.append(pubkey.hex()) self.priv.append(privkey) - if self.is_bdb_compiled(): - self.final = node2.getnewaddress() - else: - self.final = getnewdestination('bech32')[2] + + def create_wallet(self, node, wallet_name): + node.createwallet(wallet_name=wallet_name, disable_private_keys=True) + return node.get_wallet_rpc(wallet_name) def run_test(self): node0, node1, node2 = self.nodes self.wallet = MiniWallet(test_node=node0) - if self.is_bdb_compiled(): - self.import_deterministic_coinbase_privkeys() + if self.is_wallet_compiled(): self.check_addmultisigaddress_errors() self.log.info('Generating blocks ...') self.generate(self.wallet, 149) - self.moved = 0 - for self.nkeys in [3, 5]: - for self.nsigs in [2, 3]: - for self.output_type in ["bech32", "p2sh-segwit", "legacy"]: - self.get_keys() - self.do_multisig() - if self.is_bdb_compiled(): - self.checkbalances() - - # Test mixed compressed and uncompressed pubkeys - self.log.info('Mixed compressed and uncompressed multisigs are not allowed') - pk0, pk1, pk2 = [getnewdestination('bech32')[0].hex() for _ in range(3)] - - # decompress pk2 - pk_obj = ECPubKey() - pk_obj.set(bytes.fromhex(pk2)) - pk_obj.compressed = False - pk2 = pk_obj.get_bytes().hex() - - if self.is_bdb_compiled(): - node0.createwallet(wallet_name='wmulti0', disable_private_keys=True) - wmulti0 = node0.get_wallet_rpc('wmulti0') - - # Check all permutations of keys because order matters apparently - for keys in itertools.permutations([pk0, pk1, pk2]): - # Results should be the same as this legacy one - legacy_addr = node0.createmultisig(2, keys, 'legacy')['address'] - - if self.is_bdb_compiled(): - result = wmulti0.addmultisigaddress(2, keys, '', 'legacy') - assert_equal(legacy_addr, result['address']) - assert 'warnings' not in result - - # Generate addresses with the segwit types. These should all make legacy addresses - err_msg = ["Unable to make chosen address type, please ensure no uncompressed public keys are present."] - - for addr_type in ['bech32', 'p2sh-segwit']: - result = self.nodes[0].createmultisig(nrequired=2, keys=keys, address_type=addr_type) - assert_equal(legacy_addr, result['address']) - assert_equal(result['warnings'], err_msg) - - if self.is_bdb_compiled(): - result = wmulti0.addmultisigaddress(nrequired=2, keys=keys, address_type=addr_type) - assert_equal(legacy_addr, result['address']) - assert_equal(result['warnings'], err_msg) - - self.log.info('Testing sortedmulti descriptors with BIP 67 test vectors') - with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/rpc_bip67.json'), encoding='utf-8') as f: - vectors = json.load(f) + wallet_multi = self.create_wallet(node1, 'wmulti') if self._requires_wallet else None + self.create_keys(21) # max number of allowed keys + 1 + m_of_n = [(2, 3), (3, 3), (2, 5), (3, 5), (10, 15), (15, 15)] + for (sigs, keys) in m_of_n: + for output_type in ["bech32", "p2sh-segwit", "legacy"]: + self.do_multisig(keys, sigs, output_type, wallet_multi) - for t in vectors: - key_str = ','.join(t['keys']) - desc = descsum_create('sh(sortedmulti(2,{}))'.format(key_str)) - assert_equal(self.nodes[0].deriveaddresses(desc)[0], t['address']) - sorted_key_str = ','.join(t['sorted_keys']) - sorted_key_desc = descsum_create('sh(multi(2,{}))'.format(sorted_key_str)) - assert_equal(self.nodes[0].deriveaddresses(sorted_key_desc)[0], t['address']) + self.test_multisig_script_limit(wallet_multi) + self.test_mixing_uncompressed_and_compressed_keys(node0, wallet_multi) + self.test_sortedmulti_descriptors_bip67() # Check that bech32m is currently not allowed assert_raises_rpc_error(-5, "createmultisig cannot create bech32m multisig addresses", self.nodes[0].createmultisig, 2, self.pub, "bech32m") + self.log.info('Check correct encoding of multisig script for all n (1..20)') + for nkeys in range(1, 20+1): + keys = [self.pub[0]]*nkeys + expected_ms_script = keys_to_multisig_script(keys, k=nkeys) # simply use n-of-n + # note that the 'legacy' address type fails for n values larger than 15 + # due to exceeding the P2SH size limit (520 bytes), so we use 'bech32' instead + # (for the purpose of this encoding test, we don't care about the resulting address) + res = self.nodes[0].createmultisig(nrequired=nkeys, keys=keys, address_type='bech32') + assert_equal(res['redeemScript'], expected_ms_script.hex()) + def check_addmultisigaddress_errors(self): if self.options.descriptors: return @@ -133,117 +95,165 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): pubs = [self.nodes[1].getaddressinfo(addr)["pubkey"] for addr in addresses] assert_raises_rpc_error(-5, "Bech32m multisig addresses cannot be created with legacy wallets", self.nodes[0].addmultisigaddress, 2, pubs, "", "bech32m") - def checkbalances(self): - node0, node1, node2 = self.nodes - self.generate(node0, COINBASE_MATURITY) + def test_multisig_script_limit(self, wallet_multi): + node1 = self.nodes[1] + pubkeys = self.pub[0:20] - bal0 = node0.getbalance() - bal1 = node1.getbalance() - bal2 = node2.getbalance() - balw = self.wallet.get_balance() + self.log.info('Test legacy redeem script max size limit') + assert_raises_rpc_error(-8, "redeemScript exceeds size limit: 684 > 520", node1.createmultisig, 16, pubkeys, 'legacy') - height = node0.getblockchaininfo()["blocks"] - assert 150 < height < 350 - total = 149 * 50 + (height - 149 - 100) * 25 - assert bal1 == 0 - assert bal2 == self.moved - assert_equal(bal0 + bal1 + bal2 + balw, total) + self.log.info('Test valid 16-20 multisig p2sh-legacy and bech32 (no wallet)') + self.do_multisig(nkeys=20, nsigs=16, output_type="p2sh-segwit", wallet_multi=None) + self.do_multisig(nkeys=20, nsigs=16, output_type="bech32", wallet_multi=None) - def do_multisig(self): - node0, node1, node2 = self.nodes + self.log.info('Test invalid 16-21 multisig p2sh-legacy and bech32 (no wallet)') + assert_raises_rpc_error(-8, "Number of keys involved in the multisignature address creation > 20", node1.createmultisig, 16, self.pub, 'p2sh-segwit') + assert_raises_rpc_error(-8, "Number of keys involved in the multisignature address creation > 20", node1.createmultisig, 16, self.pub, 'bech32') - if self.is_bdb_compiled(): - if 'wmulti' not in node1.listwallets(): - try: - node1.loadwallet('wmulti') - except JSONRPCException as e: - path = self.nodes[1].wallets_path / "wmulti" - if e.error['code'] == -18 and "Wallet file verification failed. Failed to load database path '{}'. Path does not exist.".format(path) in e.error['message']: - node1.createwallet(wallet_name='wmulti', disable_private_keys=True) - else: - raise - wmulti = node1.get_wallet_rpc('wmulti') + # Check legacy wallet related command + self.log.info('Test legacy redeem script max size limit (with wallet)') + if wallet_multi is not None and not self.options.descriptors: + assert_raises_rpc_error(-8, "redeemScript exceeds size limit: 684 > 520", wallet_multi.addmultisigaddress, 16, pubkeys, '', 'legacy') + + self.log.info('Test legacy wallet unsupported operation. 16-20 multisig p2sh-legacy and bech32 generation') + # Due an internal limitation on legacy wallets, the redeem script limit also applies to p2sh-segwit and bech32 (even when the scripts are valid) + # We take this as a "good thing" to tell users to upgrade to descriptors. + assert_raises_rpc_error(-4, "Unsupported multisig script size for legacy wallet. Upgrade to descriptors to overcome this limitation for p2sh-segwit or bech32 scripts", wallet_multi.addmultisigaddress, 16, pubkeys, '', 'p2sh-segwit') + assert_raises_rpc_error(-4, "Unsupported multisig script size for legacy wallet. Upgrade to descriptors to overcome this limitation for p2sh-segwit or bech32 scripts", wallet_multi.addmultisigaddress, 16, pubkeys, '', 'bech32') + + def do_multisig(self, nkeys, nsigs, output_type, wallet_multi): + node0, node1, node2 = self.nodes + pub_keys = self.pub[0: nkeys] + priv_keys = self.priv[0: nkeys] # Construct the expected descriptor - desc = 'multi({},{})'.format(self.nsigs, ','.join(self.pub)) - if self.output_type == 'legacy': + desc = 'multi({},{})'.format(nsigs, ','.join(pub_keys)) + if output_type == 'legacy': desc = 'sh({})'.format(desc) - elif self.output_type == 'p2sh-segwit': + elif output_type == 'p2sh-segwit': desc = 'sh(wsh({}))'.format(desc) - elif self.output_type == 'bech32': + elif output_type == 'bech32': desc = 'wsh({})'.format(desc) desc = descsum_create(desc) - msig = node2.createmultisig(self.nsigs, self.pub, self.output_type) + msig = node2.createmultisig(nsigs, pub_keys, output_type) assert 'warnings' not in msig madd = msig["address"] mredeem = msig["redeemScript"] assert_equal(desc, msig['descriptor']) - if self.output_type == 'bech32': + if output_type == 'bech32': assert madd[0:4] == "bcrt" # actually a bech32 address - if self.is_bdb_compiled(): + if wallet_multi is not None: # compare against addmultisigaddress - msigw = wmulti.addmultisigaddress(self.nsigs, self.pub, None, self.output_type) + msigw = wallet_multi.addmultisigaddress(nsigs, pub_keys, None, output_type) maddw = msigw["address"] mredeemw = msigw["redeemScript"] assert_equal(desc, drop_origins(msigw['descriptor'])) # addmultisigiaddress and createmultisig work the same assert maddw == madd assert mredeemw == mredeem - wmulti.unloadwallet() spk = address_to_scriptpubkey(madd) - txid = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=spk, amount=1300)["txid"] - tx = node0.getrawtransaction(txid, True) - vout = [v["n"] for v in tx["vout"] if madd == v["scriptPubKey"]["address"]] - assert len(vout) == 1 - vout = vout[0] - scriptPubKey = tx["vout"][vout]["scriptPubKey"]["hex"] - value = tx["vout"][vout]["value"] - prevtxs = [{"txid": txid, "vout": vout, "scriptPubKey": scriptPubKey, "redeemScript": mredeem, "amount": value}] + value = decimal.Decimal("0.00004000") + tx = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=spk, amount=int(value * COIN)) + prevtxs = [{"txid": tx["txid"], "vout": tx["sent_vout"], "scriptPubKey": spk.hex(), "redeemScript": mredeem, "amount": value}] self.generate(node0, 1) - outval = value - decimal.Decimal("0.00001000") - rawtx = node2.createrawtransaction([{"txid": txid, "vout": vout}], [{self.final: outval}]) + outval = value - decimal.Decimal("0.00002000") # deduce fee (must be higher than the min relay fee) + # send coins to node2 when wallet is enabled + node2_balance = node2.getbalances()['mine']['trusted'] if self.is_wallet_compiled() else 0 + out_addr = node2.getnewaddress() if self.is_wallet_compiled() else getnewdestination('bech32')[2] + rawtx = node2.createrawtransaction([{"txid": tx["txid"], "vout": tx["sent_vout"]}], [{out_addr: outval}]) prevtx_err = dict(prevtxs[0]) del prevtx_err["redeemScript"] - assert_raises_rpc_error(-8, "Missing redeemScript/witnessScript", node2.signrawtransactionwithkey, rawtx, self.priv[0:self.nsigs-1], [prevtx_err]) + assert_raises_rpc_error(-8, "Missing redeemScript/witnessScript", node2.signrawtransactionwithkey, rawtx, priv_keys[0:nsigs-1], [prevtx_err]) # if witnessScript specified, all ok prevtx_err["witnessScript"] = prevtxs[0]["redeemScript"] - node2.signrawtransactionwithkey(rawtx, self.priv[0:self.nsigs-1], [prevtx_err]) + node2.signrawtransactionwithkey(rawtx, priv_keys[0:nsigs-1], [prevtx_err]) # both specified, also ok prevtx_err["redeemScript"] = prevtxs[0]["redeemScript"] - node2.signrawtransactionwithkey(rawtx, self.priv[0:self.nsigs-1], [prevtx_err]) + node2.signrawtransactionwithkey(rawtx, priv_keys[0:nsigs-1], [prevtx_err]) # redeemScript mismatch to witnessScript prevtx_err["redeemScript"] = "6a" # OP_RETURN - assert_raises_rpc_error(-8, "redeemScript does not correspond to witnessScript", node2.signrawtransactionwithkey, rawtx, self.priv[0:self.nsigs-1], [prevtx_err]) + assert_raises_rpc_error(-8, "redeemScript does not correspond to witnessScript", node2.signrawtransactionwithkey, rawtx, priv_keys[0:nsigs-1], [prevtx_err]) # redeemScript does not match scriptPubKey del prevtx_err["witnessScript"] - assert_raises_rpc_error(-8, "redeemScript/witnessScript does not match scriptPubKey", node2.signrawtransactionwithkey, rawtx, self.priv[0:self.nsigs-1], [prevtx_err]) + assert_raises_rpc_error(-8, "redeemScript/witnessScript does not match scriptPubKey", node2.signrawtransactionwithkey, rawtx, priv_keys[0:nsigs-1], [prevtx_err]) # witnessScript does not match scriptPubKey prevtx_err["witnessScript"] = prevtx_err["redeemScript"] del prevtx_err["redeemScript"] - assert_raises_rpc_error(-8, "redeemScript/witnessScript does not match scriptPubKey", node2.signrawtransactionwithkey, rawtx, self.priv[0:self.nsigs-1], [prevtx_err]) + assert_raises_rpc_error(-8, "redeemScript/witnessScript does not match scriptPubKey", node2.signrawtransactionwithkey, rawtx, priv_keys[0:nsigs-1], [prevtx_err]) - rawtx2 = node2.signrawtransactionwithkey(rawtx, self.priv[0:self.nsigs - 1], prevtxs) - rawtx3 = node2.signrawtransactionwithkey(rawtx2["hex"], [self.priv[-1]], prevtxs) + rawtx2 = node2.signrawtransactionwithkey(rawtx, priv_keys[0:nsigs - 1], prevtxs) + rawtx3 = node2.signrawtransactionwithkey(rawtx2["hex"], [priv_keys[-1]], prevtxs) + assert rawtx3['complete'] - self.moved += outval tx = node0.sendrawtransaction(rawtx3["hex"], 0) blk = self.generate(node0, 1)[0] assert tx in node0.getblock(blk)["tx"] + # When the wallet is enabled, assert node2 sees the incoming amount + if self.is_wallet_compiled(): + assert_equal(node2.getbalances()['mine']['trusted'], node2_balance + outval) + txinfo = node0.getrawtransaction(tx, True, blk) - self.log.info("n/m=%d/%d %s size=%d vsize=%d weight=%d" % (self.nsigs, self.nkeys, self.output_type, txinfo["size"], txinfo["vsize"], txinfo["weight"])) + self.log.info("n/m=%d/%d %s size=%d vsize=%d weight=%d" % (nsigs, nkeys, output_type, txinfo["size"], txinfo["vsize"], txinfo["weight"])) + + def test_mixing_uncompressed_and_compressed_keys(self, node, wallet_multi): + self.log.info('Mixed compressed and uncompressed multisigs are not allowed') + pk0, pk1, pk2 = [getnewdestination('bech32')[0].hex() for _ in range(3)] + + # decompress pk2 + pk_obj = ECPubKey() + pk_obj.set(bytes.fromhex(pk2)) + pk_obj.compressed = False + pk2 = pk_obj.get_bytes().hex() + + # Check all permutations of keys because order matters apparently + for keys in itertools.permutations([pk0, pk1, pk2]): + # Results should be the same as this legacy one + legacy_addr = node.createmultisig(2, keys, 'legacy')['address'] + + if wallet_multi is not None: + # 'addmultisigaddress' should return the same address + result = wallet_multi.addmultisigaddress(2, keys, '', 'legacy') + assert_equal(legacy_addr, result['address']) + assert 'warnings' not in result + + # Generate addresses with the segwit types. These should all make legacy addresses + err_msg = ["Unable to make chosen address type, please ensure no uncompressed public keys are present."] + + for addr_type in ['bech32', 'p2sh-segwit']: + result = self.nodes[0].createmultisig(nrequired=2, keys=keys, address_type=addr_type) + assert_equal(legacy_addr, result['address']) + assert_equal(result['warnings'], err_msg) + + if wallet_multi is not None: + result = wallet_multi.addmultisigaddress(nrequired=2, keys=keys, address_type=addr_type) + assert_equal(legacy_addr, result['address']) + assert_equal(result['warnings'], err_msg) + + def test_sortedmulti_descriptors_bip67(self): + self.log.info('Testing sortedmulti descriptors with BIP 67 test vectors') + with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/rpc_bip67.json'), encoding='utf-8') as f: + vectors = json.load(f) + + for t in vectors: + key_str = ','.join(t['keys']) + desc = descsum_create('sh(sortedmulti(2,{}))'.format(key_str)) + assert_equal(self.nodes[0].deriveaddresses(desc)[0], t['address']) + sorted_key_str = ','.join(t['sorted_keys']) + sorted_key_desc = descsum_create('sh(multi(2,{}))'.format(sorted_key_str)) + assert_equal(self.nodes[0].deriveaddresses(sorted_key_desc)[0], t['address']) if __name__ == '__main__': diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py index 1ea6cf52d1..c92c8da357 100755 --- a/test/functional/rpc_dumptxoutset.py +++ b/test/functional/rpc_dumptxoutset.py @@ -43,7 +43,7 @@ class DumptxoutsetTest(BitcoinTestFramework): # UTXO snapshot hash should be deterministic based on mocked time. assert_equal( sha256sum_file(str(expected_path)).hex(), - 'b1bacb602eacf5fbc9a7c2ef6eeb0d229c04e98bdf0c2ea5929012cd0eae3830') + '2f775f82811150d310527b5ff773f81fb0fb517e941c543c1f7c4d38fd2717b3') assert_equal( out['txoutset_hash'], 'a0b7baa3bf5ccbd3279728f230d7ca0c44a76e9923fca8f32dbfd08d65ea496a') diff --git a/test/functional/rpc_generate.py b/test/functional/rpc_generate.py index 20f62079fd..3e250925e7 100755 --- a/test/functional/rpc_generate.py +++ b/test/functional/rpc_generate.py @@ -87,7 +87,7 @@ class RPCGenerateTest(BitcoinTestFramework): txid1 = miniwallet.send_self_transfer(from_node=node)['txid'] utxo1 = miniwallet.get_utxo(txid=txid1) rawtx2 = miniwallet.create_self_transfer(utxo_to_spend=utxo1)['hex'] - assert_raises_rpc_error(-25, 'TestBlockValidity failed: bad-txns-inputs-missingorspent', self.generateblock, node, address, [rawtx2, txid1]) + assert_raises_rpc_error(-25, 'testBlockValidity failed: bad-txns-inputs-missingorspent', self.generateblock, node, address, [rawtx2, txid1]) self.log.info('Fail to generate block with txid not in mempool') missing_txid = '0000000000000000000000000000000000000000000000000000000000000000' diff --git a/test/functional/rpc_packages.py b/test/functional/rpc_packages.py index 113424c0a6..1acd586d2c 100755 --- a/test/functional/rpc_packages.py +++ b/test/functional/rpc_packages.py @@ -394,7 +394,7 @@ class RPCPackagesTest(BitcoinTestFramework): peer = node.add_p2p_connection(P2PTxInvStore()) txs = self.wallet.create_self_transfer_chain(chain_length=2) bad_child = tx_from_hex(txs[1]["hex"]) - bad_child.nVersion = -1 + bad_child.version = 0xffffffff hex_partial_acceptance = [txs[0]["hex"], bad_child.serialize().hex()] res = node.submitpackage(hex_partial_acceptance) assert_equal(res["package_msg"], "transaction failed") diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index 3978c80dde..f974a05f7b 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -463,20 +463,34 @@ class RawTransactionsTest(BitcoinTestFramework): self.log.info("Test transaction version numbers") # Test the minimum transaction version number that fits in a signed 32-bit integer. - # As transaction version is unsigned, this should convert to its unsigned equivalent. + # As transaction version is serialized unsigned, this should convert to its unsigned equivalent. tx = CTransaction() - tx.nVersion = -0x80000000 + tx.version = 0x80000000 rawtx = tx.serialize().hex() decrawtx = self.nodes[0].decoderawtransaction(rawtx) assert_equal(decrawtx['version'], 0x80000000) # Test the maximum transaction version number that fits in a signed 32-bit integer. tx = CTransaction() - tx.nVersion = 0x7fffffff + tx.version = 0x7fffffff rawtx = tx.serialize().hex() decrawtx = self.nodes[0].decoderawtransaction(rawtx) assert_equal(decrawtx['version'], 0x7fffffff) + # Test the minimum transaction version number that fits in an unsigned 32-bit integer. + tx = CTransaction() + tx.version = 0 + rawtx = tx.serialize().hex() + decrawtx = self.nodes[0].decoderawtransaction(rawtx) + assert_equal(decrawtx['version'], 0) + + # Test the maximum transaction version number that fits in an unsigned 32-bit integer. + tx = CTransaction() + tx.version = 0xffffffff + rawtx = tx.serialize().hex() + decrawtx = self.nodes[0].decoderawtransaction(rawtx) + assert_equal(decrawtx['version'], 0xffffffff) + def raw_multisig_transaction_legacy_tests(self): self.log.info("Test raw multisig transactions (legacy)") # The traditional multisig workflow does not work with descriptor wallets so these are legacy only. @@ -585,6 +599,8 @@ class RawTransactionsTest(BitcoinTestFramework): rawTxPartialSigned2 = self.nodes[2].signrawtransactionwithwallet(rawTx2, inputs) self.log.debug(rawTxPartialSigned2) assert_equal(rawTxPartialSigned2['complete'], False) # node2 only has one key, can't comp. sign the tx + assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].combinerawtransaction, [rawTxPartialSigned1['hex'], rawTxPartialSigned2['hex'] + "00"]) + assert_raises_rpc_error(-22, "Missing transactions", self.nodes[0].combinerawtransaction, []) rawTxComb = self.nodes[2].combinerawtransaction([rawTxPartialSigned1['hex'], rawTxPartialSigned2['hex']]) self.log.debug(rawTxComb) self.nodes[2].sendrawtransaction(rawTxComb) @@ -592,6 +608,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.sync_all() self.generate(self.nodes[0], 1) assert_equal(self.nodes[0].getbalance(), bal + Decimal('50.00000000') + Decimal('2.19000000')) # block reward + tx + assert_raises_rpc_error(-25, "Input not found or already spent", self.nodes[0].combinerawtransaction, [rawTxPartialSigned1['hex'], rawTxPartialSigned2['hex']]) if __name__ == '__main__': diff --git a/test/functional/rpc_signrawtransactionwithkey.py b/test/functional/rpc_signrawtransactionwithkey.py index 268584331e..f4fec13495 100755 --- a/test/functional/rpc_signrawtransactionwithkey.py +++ b/test/functional/rpc_signrawtransactionwithkey.py @@ -126,10 +126,20 @@ class SignRawTransactionWithKeyTest(BitcoinTestFramework): privkeys = [self.nodes[0].get_deterministic_priv_key().key] assert_raises_rpc_error(-8, "'all' is not a valid sighash parameter.", self.nodes[0].signrawtransactionwithkey, tx, privkeys, sighashtype="all") + def invalid_private_key_and_tx(self): + self.log.info("Test signing transaction with an invalid private key") + tx = self.nodes[0].createrawtransaction(INPUTS, OUTPUTS) + privkeys = ["123"] + assert_raises_rpc_error(-5, "Invalid private key", self.nodes[0].signrawtransactionwithkey, tx, privkeys) + self.log.info("Test signing transaction with an invalid tx hex") + privkeys = [self.nodes[0].get_deterministic_priv_key().key] + assert_raises_rpc_error(-22, "TX decode failed. Make sure the tx has at least one input.", self.nodes[0].signrawtransactionwithkey, tx + "00", privkeys) + def run_test(self): self.successful_signing_test() self.witness_script_test() self.invalid_sighashtype_test() + self.invalid_private_key_and_tx() if __name__ == '__main__': diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index 7edf9f3679..a357ae4d34 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -26,7 +26,7 @@ ServiceProxy class: - HTTP connections persist for the life of the AuthServiceProxy object (if server supports HTTP/1.1) -- sends protocol 'version', per JSON-RPC 1.1 +- sends "jsonrpc":"2.0", per JSON-RPC 2.0 - sends proper, incrementing 'id' - sends Basic HTTP authentication headers - parses all JSON numbers that look like floats as Decimal @@ -117,7 +117,7 @@ class AuthServiceProxy(): params = dict(args=args, **argsn) else: params = args or argsn - return {'version': '1.1', + return {'jsonrpc': '2.0', 'method': self._service_name, 'params': params, 'id': AuthServiceProxy.__id_count} @@ -125,15 +125,28 @@ class AuthServiceProxy(): def __call__(self, *args, **argsn): postdata = json.dumps(self.get_request(*args, **argsn), default=serialization_fallback, ensure_ascii=self.ensure_ascii) response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) - if response['error'] is not None: - raise JSONRPCException(response['error'], status) - elif 'result' not in response: - raise JSONRPCException({ - 'code': -343, 'message': 'missing JSON-RPC result'}, status) - elif status != HTTPStatus.OK: - raise JSONRPCException({ - 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) + # For backwards compatibility tests, accept JSON RPC 1.1 responses + if 'jsonrpc' not in response: + if response['error'] is not None: + raise JSONRPCException(response['error'], status) + elif 'result' not in response: + raise JSONRPCException({ + 'code': -343, 'message': 'missing JSON-RPC result'}, status) + elif status != HTTPStatus.OK: + raise JSONRPCException({ + 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) + else: + return response['result'] else: + assert response['jsonrpc'] == '2.0' + if status != HTTPStatus.OK: + raise JSONRPCException({ + 'code': -342, 'message': 'non-200 HTTP status code'}, status) + if 'error' in response: + raise JSONRPCException(response['error'], status) + elif 'result' not in response: + raise JSONRPCException({ + 'code': -343, 'message': 'missing JSON-RPC 2.0 result and error'}, status) return response['result'] def batch(self, rpc_call_list): @@ -142,7 +155,7 @@ class AuthServiceProxy(): response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) if status != HTTPStatus.OK: raise JSONRPCException({ - 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) + 'code': -342, 'message': 'non-200 HTTP status code'}, status) return response def _get_response(self): diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 4e496a9275..005f7546a8 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -560,12 +560,12 @@ class CTxWitness: class CTransaction: - __slots__ = ("hash", "nLockTime", "nVersion", "sha256", "vin", "vout", + __slots__ = ("hash", "nLockTime", "version", "sha256", "vin", "vout", "wit") def __init__(self, tx=None): if tx is None: - self.nVersion = 2 + self.version = 2 self.vin = [] self.vout = [] self.wit = CTxWitness() @@ -573,7 +573,7 @@ class CTransaction: self.sha256 = None self.hash = None else: - self.nVersion = tx.nVersion + self.version = tx.version self.vin = copy.deepcopy(tx.vin) self.vout = copy.deepcopy(tx.vout) self.nLockTime = tx.nLockTime @@ -582,7 +582,7 @@ class CTransaction: self.wit = copy.deepcopy(tx.wit) def deserialize(self, f): - self.nVersion = int.from_bytes(f.read(4), "little", signed=True) + self.version = int.from_bytes(f.read(4), "little") self.vin = deser_vector(f, CTxIn) flags = 0 if len(self.vin) == 0: @@ -605,7 +605,7 @@ class CTransaction: def serialize_without_witness(self): r = b"" - r += self.nVersion.to_bytes(4, "little", signed=True) + r += self.version.to_bytes(4, "little") r += ser_vector(self.vin) r += ser_vector(self.vout) r += self.nLockTime.to_bytes(4, "little") @@ -617,7 +617,7 @@ class CTransaction: if not self.wit.is_null(): flags |= 1 r = b"" - r += self.nVersion.to_bytes(4, "little", signed=True) + r += self.version.to_bytes(4, "little") if flags: dummy = [] r += ser_vector(dummy) @@ -677,8 +677,8 @@ class CTransaction: return math.ceil(self.get_weight() / WITNESS_SCALE_FACTOR) def __repr__(self): - return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \ - % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime) + return "CTransaction(version=%i vin=%s vout=%s wit=%s nLockTime=%i)" \ + % (self.version, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime) class CBlockHeader: diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index 00bd1e4017..4b846df94a 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -411,7 +411,7 @@ class P2PConnection(asyncio.Protocol): tmsg = self.magic_bytes tmsg += msgtype tmsg += b"\x00" * (12 - len(msgtype)) - tmsg += struct.pack("<I", len(data)) + tmsg += len(data).to_bytes(4, "little") th = sha256(data) h = sha256(th) tmsg += h[:4] diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 7b19d31e17..97d62f957b 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -8,7 +8,6 @@ This file is modified from python-bitcoinlib. """ from collections import namedtuple -import struct import unittest from .key import TaggedHash, tweak_add_pubkey, compute_xonly_pubkey @@ -58,9 +57,9 @@ class CScriptOp(int): elif len(d) <= 0xff: return b'\x4c' + bytes([len(d)]) + d # OP_PUSHDATA1 elif len(d) <= 0xffff: - return b'\x4d' + struct.pack(b'<H', len(d)) + d # OP_PUSHDATA2 + return b'\x4d' + len(d).to_bytes(2, "little") + d # OP_PUSHDATA2 elif len(d) <= 0xffffffff: - return b'\x4e' + struct.pack(b'<I', len(d)) + d # OP_PUSHDATA4 + return b'\x4e' + len(d).to_bytes(4, "little") + d # OP_PUSHDATA4 else: raise ValueError("Data too long to encode in a PUSHDATA op") @@ -670,7 +669,7 @@ def LegacySignatureMsg(script, txTo, inIdx, hashtype): txtmp.vin.append(tmp) s = txtmp.serialize_without_witness() - s += struct.pack(b"<I", hashtype) + s += hashtype.to_bytes(4, "little") return (s, None) @@ -726,7 +725,7 @@ def SegwitV0SignatureMsg(script, txTo, inIdx, hashtype, amount): if (not (hashtype & SIGHASH_ANYONECANPAY) and (hashtype & 0x1f) != SIGHASH_SINGLE and (hashtype & 0x1f) != SIGHASH_NONE): serialize_sequence = bytes() for i in txTo.vin: - serialize_sequence += struct.pack("<I", i.nSequence) + serialize_sequence += i.nSequence.to_bytes(4, "little") hashSequence = uint256_from_str(hash256(serialize_sequence)) if ((hashtype & 0x1f) != SIGHASH_SINGLE and (hashtype & 0x1f) != SIGHASH_NONE): @@ -739,16 +738,16 @@ def SegwitV0SignatureMsg(script, txTo, inIdx, hashtype, amount): hashOutputs = uint256_from_str(hash256(serialize_outputs)) ss = bytes() - ss += struct.pack("<i", txTo.nVersion) + ss += txTo.version.to_bytes(4, "little") ss += ser_uint256(hashPrevouts) ss += ser_uint256(hashSequence) ss += txTo.vin[inIdx].prevout.serialize() ss += ser_string(script) - ss += struct.pack("<q", amount) - ss += struct.pack("<I", txTo.vin[inIdx].nSequence) + ss += amount.to_bytes(8, "little", signed=True) + ss += txTo.vin[inIdx].nSequence.to_bytes(4, "little") ss += ser_uint256(hashOutputs) ss += txTo.nLockTime.to_bytes(4, "little") - ss += struct.pack("<I", hashtype) + ss += hashtype.to_bytes(4, "little") return ss def SegwitV0SignatureHash(*args, **kwargs): @@ -800,13 +799,13 @@ def BIP341_sha_prevouts(txTo): return sha256(b"".join(i.prevout.serialize() for i in txTo.vin)) def BIP341_sha_amounts(spent_utxos): - return sha256(b"".join(struct.pack("<q", u.nValue) for u in spent_utxos)) + return sha256(b"".join(u.nValue.to_bytes(8, "little", signed=True) for u in spent_utxos)) def BIP341_sha_scriptpubkeys(spent_utxos): return sha256(b"".join(ser_string(u.scriptPubKey) for u in spent_utxos)) def BIP341_sha_sequences(txTo): - return sha256(b"".join(struct.pack("<I", i.nSequence) for i in txTo.vin)) + return sha256(b"".join(i.nSequence.to_bytes(4, "little") for i in txTo.vin)) def BIP341_sha_outputs(txTo): return sha256(b"".join(o.serialize() for o in txTo.vout)) @@ -818,8 +817,8 @@ def TaprootSignatureMsg(txTo, spent_utxos, hash_type, input_index = 0, scriptpat in_type = hash_type & SIGHASH_ANYONECANPAY spk = spent_utxos[input_index].scriptPubKey ss = bytes([0, hash_type]) # epoch, hash_type - ss += struct.pack("<i", txTo.nVersion) - ss += struct.pack("<I", txTo.nLockTime) + ss += txTo.version.to_bytes(4, "little") + ss += txTo.nLockTime.to_bytes(4, "little") if in_type != SIGHASH_ANYONECANPAY: ss += BIP341_sha_prevouts(txTo) ss += BIP341_sha_amounts(spent_utxos) @@ -835,11 +834,11 @@ def TaprootSignatureMsg(txTo, spent_utxos, hash_type, input_index = 0, scriptpat ss += bytes([spend_type]) if in_type == SIGHASH_ANYONECANPAY: ss += txTo.vin[input_index].prevout.serialize() - ss += struct.pack("<q", spent_utxos[input_index].nValue) + ss += spent_utxos[input_index].nValue.to_bytes(8, "little", signed=True) ss += ser_string(spk) - ss += struct.pack("<I", txTo.vin[input_index].nSequence) + ss += txTo.vin[input_index].nSequence.to_bytes(4, "little") else: - ss += struct.pack("<I", input_index) + ss += input_index.to_bytes(4, "little") if (spend_type & 1): ss += sha256(ser_string(annex)) if out_type == SIGHASH_SINGLE: @@ -850,7 +849,7 @@ def TaprootSignatureMsg(txTo, spent_utxos, hash_type, input_index = 0, scriptpat if (scriptpath): ss += TaggedHash("TapLeaf", bytes([leaf_ver]) + ser_string(script)) ss += bytes([0]) - ss += struct.pack("<i", codeseparator_pos) + ss += codeseparator_pos.to_bytes(4, "little", signed=True) assert len(ss) == 175 - (in_type == SIGHASH_ANYONECANPAY) * 49 - (out_type != SIGHASH_ALL and out_type != SIGHASH_SINGLE) * 32 + (annex is not None) * 32 + scriptpath * 37 return ss diff --git a/test/functional/test_framework/script_util.py b/test/functional/test_framework/script_util.py index 62894cc0f4..855f3b8cf5 100755 --- a/test/functional/test_framework/script_util.py +++ b/test/functional/test_framework/script_util.py @@ -3,10 +3,13 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Useful Script constants and utils.""" +import unittest + from test_framework.script import ( CScript, - CScriptOp, OP_0, + OP_15, + OP_16, OP_CHECKMULTISIG, OP_CHECKSIG, OP_DUP, @@ -49,10 +52,8 @@ def keys_to_multisig_script(keys, *, k=None): if k is None: # n-of-n multisig by default k = n assert k <= n - op_k = CScriptOp.encode_op_n(k) - op_n = CScriptOp.encode_op_n(n) checked_keys = [check_key(key) for key in keys] - return CScript([op_k] + checked_keys + [op_n, OP_CHECKMULTISIG]) + return CScript([k] + checked_keys + [n, OP_CHECKMULTISIG]) def keyhash_to_p2pkh_script(hash): @@ -125,3 +126,19 @@ def check_script(script): if isinstance(script, bytes) or isinstance(script, CScript): return script assert False + + +class TestFrameworkScriptUtil(unittest.TestCase): + def test_multisig(self): + fake_pubkey = bytes([0]*33) + # check correct encoding of P2MS script with n,k <= 16 + normal_ms_script = keys_to_multisig_script([fake_pubkey]*16, k=15) + self.assertEqual(len(normal_ms_script), 1 + 16*34 + 1 + 1) + self.assertTrue(normal_ms_script.startswith(bytes([OP_15]))) + self.assertTrue(normal_ms_script.endswith(bytes([OP_16, OP_CHECKMULTISIG]))) + + # check correct encoding of P2MS script with n,k > 16 + max_ms_script = keys_to_multisig_script([fake_pubkey]*20, k=19) + self.assertEqual(len(max_ms_script), 2 + 20*34 + 2 + 1) + self.assertTrue(max_ms_script.startswith(bytes([1, 19]))) # using OP_PUSH1 + self.assertTrue(max_ms_script.endswith(bytes([1, 20, OP_CHECKMULTISIG]))) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index a2f767cc98..9e44a11143 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2022 The Bitcoin Core developers +# Copyright (c) 2014-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Base class for RPC testing.""" @@ -444,6 +444,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): n.createwallet(wallet_name=wallet_name, descriptors=self.options.descriptors, load_on_startup=True) n.importprivkey(privkey=n.get_deterministic_priv_key().key, label='coinbase', rescan=True) + # Only enables wallet support when the module is available + def enable_wallet_if_possible(self): + self._requires_wallet = self.is_wallet_compiled() + def run_test(self): """Tests must override this method to define test logic""" raise NotImplementedError @@ -610,8 +614,6 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): """ from_connection = self.nodes[a] to_connection = self.nodes[b] - from_num_peers = 1 + len(from_connection.getpeerinfo()) - to_num_peers = 1 + len(to_connection.getpeerinfo()) ip_port = "127.0.0.1:" + str(p2p_port(b)) if peer_advertises_v2 is None: @@ -627,19 +629,28 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): if not wait_for_connect: return - # poll until version handshake complete to avoid race conditions - # with transaction relaying - # See comments in net_processing: - # * Must have a version message before anything else - # * Must have a verack message before anything else - self.wait_until(lambda: sum(peer['version'] != 0 for peer in from_connection.getpeerinfo()) == from_num_peers) - self.wait_until(lambda: sum(peer['version'] != 0 for peer in to_connection.getpeerinfo()) == to_num_peers) - self.wait_until(lambda: sum(peer['bytesrecv_per_msg'].pop('verack', 0) >= 21 for peer in from_connection.getpeerinfo()) == from_num_peers) - self.wait_until(lambda: sum(peer['bytesrecv_per_msg'].pop('verack', 0) >= 21 for peer in to_connection.getpeerinfo()) == to_num_peers) - # The message bytes are counted before processing the message, so make - # sure it was fully processed by waiting for a ping. - self.wait_until(lambda: sum(peer["bytesrecv_per_msg"].pop("pong", 0) >= 29 for peer in from_connection.getpeerinfo()) == from_num_peers) - self.wait_until(lambda: sum(peer["bytesrecv_per_msg"].pop("pong", 0) >= 29 for peer in to_connection.getpeerinfo()) == to_num_peers) + # Use subversion as peer id. Test nodes have their node number appended to the user agent string + from_connection_subver = from_connection.getnetworkinfo()['subversion'] + to_connection_subver = to_connection.getnetworkinfo()['subversion'] + + def find_conn(node, peer_subversion, inbound): + return next(filter(lambda peer: peer['subver'] == peer_subversion and peer['inbound'] == inbound, node.getpeerinfo()), None) + + self.wait_until(lambda: find_conn(from_connection, to_connection_subver, inbound=False) is not None) + self.wait_until(lambda: find_conn(to_connection, from_connection_subver, inbound=True) is not None) + + def check_bytesrecv(peer, msg_type, min_bytes_recv): + assert peer is not None, "Error: peer disconnected" + return peer['bytesrecv_per_msg'].pop(msg_type, 0) >= min_bytes_recv + + # Poll until version handshake (fSuccessfullyConnected) is complete to + # avoid race conditions, because some message types are blocked from + # being sent or received before fSuccessfullyConnected. + # + # As the flag fSuccessfullyConnected is not exposed, check it by + # waiting for a pong, which can only happen after the flag was set. + self.wait_until(lambda: check_bytesrecv(find_conn(from_connection, to_connection_subver, inbound=False), 'pong', 29)) + self.wait_until(lambda: check_bytesrecv(find_conn(to_connection, from_connection_subver, inbound=True), 'pong', 29)) def disconnect_nodes(self, a, b): def disconnect_nodes_helper(node_a, node_b): diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 4ba92a7b1f..0ac0af27d5 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -106,7 +106,7 @@ class TestNode(): "-debugexclude=libevent", "-debugexclude=leveldb", "-debugexclude=rand", - "-uacomment=testnode%d" % i, + "-uacomment=testnode%d" % i, # required for subversion uniqueness across peers ] if self.descriptors is None: self.args.append("-disablewallet") @@ -241,7 +241,7 @@ class TestNode(): if self.start_perf: self._start_perf() - def wait_for_rpc_connection(self): + def wait_for_rpc_connection(self, *, wait_for_import=True): """Sets up an RPC connection to the bitcoind process. Returns False if unable to connect.""" # Poll at a rate of four times per second poll_per_s = 4 @@ -263,7 +263,7 @@ class TestNode(): ) rpc.getblockcount() # If the call to getblockcount() succeeds then the RPC connection is up - if self.version_is_at_least(190000): + if self.version_is_at_least(190000) and wait_for_import: # getmempoolinfo.loaded is available since commit # bb8ae2c (version 0.19.0) self.wait_until(lambda: rpc.getmempoolinfo()['loaded']) diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index 4433cbcc55..cb0d291361 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -7,6 +7,7 @@ from copy import deepcopy from decimal import Decimal from enum import Enum +import math from typing import ( Any, Optional, @@ -33,10 +34,13 @@ from test_framework.messages import ( CTxInWitness, CTxOut, hash256, + ser_compact_size, + WITNESS_SCALE_FACTOR, ) from test_framework.script import ( CScript, LEAF_VERSION_TAPSCRIPT, + OP_1, OP_NOP, OP_RETURN, OP_TRUE, @@ -52,6 +56,7 @@ from test_framework.script_util import ( from test_framework.util import ( assert_equal, assert_greater_than_or_equal, + get_fee, ) from test_framework.wallet_util import generate_keypair @@ -119,13 +124,16 @@ class MiniWallet: """Pad a transaction with extra outputs until it reaches a target weight (or higher). returns the tx """ - tx.vout.append(CTxOut(nValue=0, scriptPubKey=CScript([OP_RETURN, b'a']))) + tx.vout.append(CTxOut(nValue=0, scriptPubKey=CScript([OP_RETURN]))) + # determine number of needed padding bytes by converting weight difference to vbytes dummy_vbytes = (target_weight - tx.get_weight() + 3) // 4 - tx.vout[-1].scriptPubKey = CScript([OP_RETURN, b'a' * dummy_vbytes]) - # Lower bound should always be off by at most 3 + # compensate for the increase of the compact-size encoded script length + # (note that the length encoding of the unpadded output script needs one byte) + dummy_vbytes -= len(ser_compact_size(dummy_vbytes)) - 1 + tx.vout[-1].scriptPubKey = CScript([OP_RETURN] + [OP_1] * dummy_vbytes) + # Actual weight should be at most 3 higher than target weight assert_greater_than_or_equal(tx.get_weight(), target_weight) - # Higher bound should always be off by at most 3 + 12 weight (for encoding the length) - assert_greater_than_or_equal(target_weight + 15, tx.get_weight()) + assert_greater_than_or_equal(target_weight + 3, tx.get_weight()) def get_balance(self): return sum(u['value'] for u in self._utxos) @@ -321,7 +329,7 @@ class MiniWallet: tx = CTransaction() tx.vin = [CTxIn(COutPoint(int(utxo_to_spend['txid'], 16), utxo_to_spend['vout']), nSequence=seq) for utxo_to_spend, seq in zip(utxos_to_spend, sequence)] tx.vout = [CTxOut(amount_per_output, bytearray(self._scriptPubKey)) for _ in range(num_outputs)] - tx.nVersion = version + tx.version = version tx.nLockTime = locktime self.sign_tx(tx) @@ -367,6 +375,10 @@ class MiniWallet: vsize = Decimal(168) # P2PK (73 bytes scriptSig + 35 bytes scriptPubKey + 60 bytes other) else: assert False + if target_weight and not fee: # respect fee_rate if target weight is passed + # the actual weight might be off by 3 WUs, so calculate based on that (see self._bulk_tx) + max_actual_weight = target_weight + 3 + fee = get_fee(math.ceil(max_actual_weight / WITNESS_SCALE_FACTOR), fee_rate) send_value = utxo_to_spend["value"] - (fee or (fee_rate * vsize / 1000)) # create tx diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 725b116281..8475dc5faa 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -15,8 +15,10 @@ For a description of arguments recognized by test scripts, see import argparse from collections import deque import configparser +import csv import datetime import os +import pathlib import platform import time import shutil @@ -280,6 +282,7 @@ BASE_SCRIPTS = [ 'mempool_packages.py', 'mempool_package_onemore.py', 'mempool_package_limits.py', + 'mempool_package_rbf.py', 'feature_versionbits_warning.py', 'rpc_preciousblock.py', 'wallet_importprunedfunds.py --legacy-wallet', @@ -361,6 +364,7 @@ BASE_SCRIPTS = [ 'feature_addrman.py', 'feature_asmap.py', 'feature_fastprune.py', + 'feature_framework_miniwallet.py', 'mempool_unbroadcast.py', 'mempool_compatibility.py', 'mempool_accept_wtxid.py', @@ -438,6 +442,7 @@ def main(): parser.add_argument('--filter', help='filter scripts to run by regular expression') parser.add_argument("--nocleanup", dest="nocleanup", default=False, action="store_true", help="Leave bitcoinds and test.* datadir on exit or error") + parser.add_argument('--resultsfile', '-r', help='store test results (as CSV) to the provided file') args, unknown_args = parser.parse_known_args() @@ -470,6 +475,13 @@ def main(): logging.debug("Temporary test directory at %s" % tmpdir) + results_filepath = None + if args.resultsfile: + results_filepath = pathlib.Path(args.resultsfile) + # Stop early if the parent directory doesn't exist + assert results_filepath.parent.exists(), "Results file parent directory does not exist" + logging.debug("Test results will be written to " + str(results_filepath)) + enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND") if not enable_bitcoind: @@ -556,9 +568,10 @@ def main(): combined_logs_len=args.combinedlogslen, failfast=args.failfast, use_term_control=args.ansi, + results_filepath=results_filepath, ) -def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage=False, args=None, combined_logs_len=0, failfast=False, use_term_control): +def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage=False, args=None, combined_logs_len=0, failfast=False, use_term_control, results_filepath=None): args = args or [] # Warn if bitcoind is already running @@ -650,11 +663,14 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= break if "[Errno 28] No space left on device" in stdout: - sys.exit(f"Early exiting after test failure due to insuffient free space in {tmpdir}\n" + sys.exit(f"Early exiting after test failure due to insufficient free space in {tmpdir}\n" f"Test execution data left in {tmpdir}.\n" f"Additional storage is needed to execute testing.") - print_results(test_results, max_len_name, (int(time.time() - start_time))) + runtime = int(time.time() - start_time) + print_results(test_results, max_len_name, runtime) + if results_filepath: + write_results(test_results, results_filepath, runtime) if coverage: coverage_passed = coverage.report_rpc_coverage() @@ -701,6 +717,17 @@ def print_results(test_results, max_len_name, runtime): results += "Runtime: %s s\n" % (runtime) print(results) + +def write_results(test_results, filepath, total_runtime): + with open(filepath, mode="w", encoding="utf8") as results_file: + results_writer = csv.writer(results_file) + results_writer.writerow(['test', 'status', 'duration(seconds)']) + all_passed = True + for test_result in test_results: + all_passed = all_passed and test_result.was_successful + results_writer.writerow([test_result.name, test_result.status, str(test_result.time)]) + results_writer.writerow(['ALL', ("Passed" if all_passed else "Failed"), str(total_runtime)]) + class TestHandler: """ Trigger the test scripts passed in via the list. diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index c322ae52c1..2c85773bf3 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet balance RPC methods.""" from decimal import Decimal -import struct from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE as ADDRESS_WATCHONLY from test_framework.blocktools import COINBASE_MATURITY @@ -266,8 +265,8 @@ class WalletTest(BitcoinTestFramework): tx_orig = self.nodes[0].gettransaction(txid)['hex'] # Increase fee by 1 coin tx_replace = tx_orig.replace( - struct.pack("<q", 99 * 10**8).hex(), - struct.pack("<q", 98 * 10**8).hex(), + (99 * 10**8).to_bytes(8, "little", signed=True).hex(), + (98 * 10**8).to_bytes(8, "little", signed=True).hex(), ) tx_replace = self.nodes[0].signrawtransactionwithwallet(tx_replace)['hex'] # Total balance is given by the sum of outputs of the tx diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 5b7db55f45..6d45adc823 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -117,6 +117,7 @@ class BumpFeeTest(BitcoinTestFramework): # Context independent tests test_feerate_checks_replaced_outputs(self, rbf_node, peer_node) + test_bumpfee_with_feerate_ignores_walletincrementalrelayfee(self, rbf_node, peer_node) def test_invalid_parameters(self, rbf_node, peer_node, dest_address): self.log.info('Test invalid parameters') @@ -816,7 +817,7 @@ def test_feerate_checks_replaced_outputs(self, rbf_node, peer_node): # Since the bumped tx will replace all of the outputs with a single output, we can estimate that its size will 31 * (len(outputs) - 1) bytes smaller tx_size = tx_details["decoded"]["vsize"] est_bumped_size = tx_size - (len(tx_details["decoded"]["vout"]) - 1) * 31 - inc_fee_rate = max(rbf_node.getmempoolinfo()["incrementalrelayfee"], Decimal(0.00005000)) # Wallet has a fixed incremental relay fee of 5 sat/vb + inc_fee_rate = rbf_node.getmempoolinfo()["incrementalrelayfee"] # RPC gives us fee as negative min_fee = (-tx_details["fee"] + get_fee(est_bumped_size, inc_fee_rate)) * Decimal(1e8) min_fee_rate = (min_fee / est_bumped_size).quantize(Decimal("1.000")) @@ -830,5 +831,27 @@ def test_feerate_checks_replaced_outputs(self, rbf_node, peer_node): self.clear_mempool() +def test_bumpfee_with_feerate_ignores_walletincrementalrelayfee(self, rbf_node, peer_node): + self.log.info('Test that bumpfee with fee_rate ignores walletincrementalrelayfee') + # Make sure there is enough balance + peer_node.sendtoaddress(rbf_node.getnewaddress(), 2) + self.generate(peer_node, 1) + + dest_address = peer_node.getnewaddress(address_type="bech32") + tx = rbf_node.send(outputs=[{dest_address: 1}], fee_rate=2) + + # Ensure you can not fee bump with a fee_rate below or equal to the original fee_rate + assert_raises_rpc_error(-8, "Insufficient total fee", rbf_node.bumpfee, tx["txid"], {"fee_rate": 1}) + assert_raises_rpc_error(-8, "Insufficient total fee", rbf_node.bumpfee, tx["txid"], {"fee_rate": 2}) + + # Ensure you can not fee bump if the fee_rate is more than original fee_rate but the total fee from new fee_rate is + # less than (original fee + incrementalrelayfee) + assert_raises_rpc_error(-8, "Insufficient total fee", rbf_node.bumpfee, tx["txid"], {"fee_rate": 2.8}) + + # You can fee bump as long as the new fee set from fee_rate is atleast (original fee + incrementalrelayfee) + rbf_node.bumpfee(tx["txid"], {"fee_rate": 3}) + self.clear_mempool() + + if __name__ == "__main__": BumpFeeTest().main() diff --git a/test/functional/wallet_create_tx.py b/test/functional/wallet_create_tx.py index 4e31b48ec0..fa3e920c25 100755 --- a/test/functional/wallet_create_tx.py +++ b/test/functional/wallet_create_tx.py @@ -3,6 +3,9 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +from test_framework.messages import ( + tx_from_hex, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -33,6 +36,7 @@ class CreateTxWalletTest(BitcoinTestFramework): self.test_anti_fee_sniping() self.test_tx_size_too_large() self.test_create_too_long_mempool_chain() + self.test_version3() def test_anti_fee_sniping(self): self.log.info('Check that we have some (old) blocks and that anti-fee-sniping is disabled') @@ -106,6 +110,23 @@ class CreateTxWalletTest(BitcoinTestFramework): test_wallet.unloadwallet() + def test_version3(self): + self.log.info('Check wallet does not create transactions with version=3 yet') + wallet_rpc = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + + self.nodes[0].createwallet("v3") + wallet_v3 = self.nodes[0].get_wallet_rpc("v3") + + tx_data = wallet_rpc.send(outputs=[{wallet_v3.getnewaddress(): 25}], options={"change_position": 0}) + wallet_tx_data = wallet_rpc.gettransaction(tx_data["txid"]) + tx_current_version = tx_from_hex(wallet_tx_data["hex"]) + + # While v3 transactions are standard, the CURRENT_VERSION is 2. + # This test can be removed if CURRENT_VERSION is changed, and replaced with tests that the + # wallet handles v3 rules properly. + assert_equal(tx_current_version.version, 2) + wallet_v3.unloadwallet() + if __name__ == '__main__': CreateTxWalletTest().main() diff --git a/test/functional/wallet_listsinceblock.py b/test/functional/wallet_listsinceblock.py index fd586d546e..15214539a9 100755 --- a/test/functional/wallet_listsinceblock.py +++ b/test/functional/wallet_listsinceblock.py @@ -40,6 +40,7 @@ class ListSinceBlockTest(BitcoinTestFramework): self.test_no_blockhash() self.test_invalid_blockhash() self.test_reorg() + self.test_cant_read_block() self.test_double_spend() self.test_double_send() self.double_spends_filtered() @@ -167,6 +168,31 @@ class ListSinceBlockTest(BitcoinTestFramework): found = next(tx for tx in transactions if tx['txid'] == senttx) assert_equal(found['blockheight'], self.nodes[0].getblockheader(nodes2_first_blockhash)['height']) + def test_cant_read_block(self): + self.log.info('Test the RPC error "Can\'t read block from disk"') + + # Split network into two + self.split_network() + + # generate on both sides + nodes1_last_blockhash = self.generate(self.nodes[1], 6, sync_fun=lambda: self.sync_all(self.nodes[:2]))[-1] + self.generate(self.nodes[2], 7, sync_fun=lambda: self.sync_all(self.nodes[2:]))[0] + + self.join_network() + + # Renaming the block file to induce unsuccessful block read + blk_dat = (self.nodes[0].blocks_path / "blk00000.dat") + blk_dat_moved = blk_dat.rename(self.nodes[0].blocks_path / "blk00000.dat.moved") + assert not blk_dat.exists() + + # listsinceblock(nodes1_last_blockhash) should now fail as blocks are not accessible + assert_raises_rpc_error(-32603, "Can't read block from disk", + self.nodes[0].listsinceblock, nodes1_last_blockhash) + + # Restoring block file + blk_dat_moved.rename(self.nodes[0].blocks_path / "blk00000.dat") + assert blk_dat.exists() + def test_double_spend(self): ''' This tests the case where the same UTXO is spent twice on two separate diff --git a/test/functional/wallet_sendall.py b/test/functional/wallet_sendall.py index c2b800df21..1d308c225d 100755 --- a/test/functional/wallet_sendall.py +++ b/test/functional/wallet_sendall.py @@ -379,6 +379,64 @@ class SendallTest(BitcoinTestFramework): assert_equal(len(self.wallet.listunspent()), 1) assert_equal(self.wallet.listunspent()[0]['confirmations'], 6) + @cleanup + def sendall_spends_unconfirmed_change(self): + self.log.info("Test that sendall spends unconfirmed change") + self.add_utxos([17]) + self.wallet.sendtoaddress(self.remainder_target, 10) + assert_greater_than(self.wallet.getbalances()["mine"]["trusted"], 6) + self.test_sendall_success(sendall_args = [self.remainder_target]) + + assert_equal(self.wallet.getbalance(), 0) + + @cleanup + def sendall_spends_unconfirmed_inputs_if_specified(self): + self.log.info("Test that sendall spends specified unconfirmed inputs") + self.def_wallet.sendtoaddress(self.wallet.getnewaddress(), 17) + self.wallet.syncwithvalidationinterfacequeue() + assert_equal(self.wallet.getbalances()["mine"]["untrusted_pending"], 17) + unspent = self.wallet.listunspent(minconf=0)[0] + + self.wallet.sendall(recipients=[self.remainder_target], inputs=[unspent]) + assert_equal(self.wallet.getbalance(), 0) + + @cleanup + def sendall_does_ancestor_aware_funding(self): + self.log.info("Test that sendall does ancestor aware funding for unconfirmed inputs") + + # higher parent feerate + self.def_wallet.sendtoaddress(address=self.wallet.getnewaddress(), amount=17, fee_rate=20) + self.wallet.syncwithvalidationinterfacequeue() + + assert_equal(self.wallet.getbalances()["mine"]["untrusted_pending"], 17) + unspent = self.wallet.listunspent(minconf=0)[0] + + parent_txid = unspent["txid"] + assert_equal(self.wallet.gettransaction(parent_txid)["confirmations"], 0) + + res_1 = self.wallet.sendall(recipients=[self.def_wallet.getnewaddress()], inputs=[unspent], fee_rate=20, add_to_wallet=False, lock_unspents=True) + child_hex = res_1["hex"] + + child_tx = self.wallet.decoderawtransaction(child_hex) + higher_parent_feerate_amount = child_tx["vout"][0]["value"] + + # lower parent feerate + self.def_wallet.sendtoaddress(address=self.wallet.getnewaddress(), amount=17, fee_rate=10) + self.wallet.syncwithvalidationinterfacequeue() + assert_equal(self.wallet.getbalances()["mine"]["untrusted_pending"], 34) + unspent = self.wallet.listunspent(minconf=0)[0] + + parent_txid = unspent["txid"] + assert_equal(self.wallet.gettransaction(parent_txid)["confirmations"], 0) + + res_2 = self.wallet.sendall(recipients=[self.def_wallet.getnewaddress()], inputs=[unspent], fee_rate=20, add_to_wallet=False, lock_unspents=True) + child_hex = res_2["hex"] + + child_tx = self.wallet.decoderawtransaction(child_hex) + lower_parent_feerate_amount = child_tx["vout"][0]["value"] + + assert_greater_than(higher_parent_feerate_amount, lower_parent_feerate_amount) + # This tests needs to be the last one otherwise @cleanup will fail with "Transaction too large" error def sendall_fails_with_transaction_too_large(self): self.log.info("Test that sendall fails if resulting transaction is too large") @@ -460,6 +518,15 @@ class SendallTest(BitcoinTestFramework): # Sendall only uses outputs with less than a given number of confirmation when using minconf self.sendall_with_maxconf() + # Sendall spends unconfirmed change + self.sendall_spends_unconfirmed_change() + + # Sendall spends unconfirmed inputs if they are specified + self.sendall_spends_unconfirmed_inputs_if_specified() + + # Sendall does ancestor aware funding when spending an unconfirmed UTXO + self.sendall_does_ancestor_aware_funding() + # Sendall fails when many inputs result to too large transaction self.sendall_fails_with_transaction_too_large() diff --git a/test/lint/README.md b/test/lint/README.md index 13c2099808..04a836c4d2 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -26,6 +26,21 @@ Then you can use: ( cd ./test/lint/test_runner/ && cargo fmt && cargo clippy && RUST_BACKTRACE=1 cargo run ) ``` +If you wish to run individual lint checks, run the test_runner with +`--lint=TEST_TO_RUN` arguments. If running with `cargo run`, arguments after +`--` are passed to the binary you are running e.g.: + +```sh +( cd ./test/lint/test_runner/ && RUST_BACKTRACE=1 cargo run -- --lint=doc --lint=trailing_whitespace ) +``` + +To see a list of all individual lint checks available in test_runner, use `-h` +or `--help`: + +```sh +( cd ./test/lint/test_runner/ && RUST_BACKTRACE=1 cargo run -- --help ) +``` + #### Dependencies | Lint test | Dependency | @@ -37,6 +52,7 @@ Then you can use: | [`lint-python-dead-code.py`](/test/lint/lint-python-dead-code.py) | [vulture](https://github.com/jendrikseipp/vulture) | [`lint-shell.py`](/test/lint/lint-shell.py) | [ShellCheck](https://github.com/koalaman/shellcheck) | [`lint-spelling.py`](/test/lint/lint-spelling.py) | [codespell](https://github.com/codespell-project/codespell) +| markdown link check | [mlc](https://github.com/becheran/mlc) In use versions and install instructions are available in the [CI setup](../../ci/lint/04_install.sh). diff --git a/test/lint/test_runner/src/main.rs b/test/lint/test_runner/src/main.rs index d5dd98effe..9c35898c1f 100644 --- a/test/lint/test_runner/src/main.rs +++ b/test/lint/test_runner/src/main.rs @@ -4,14 +4,121 @@ use std::env; use std::fs; +use std::io::ErrorKind; use std::path::{Path, PathBuf}; -use std::process::Command; -use std::process::ExitCode; +use std::process::{Command, ExitCode, Stdio}; type LintError = String; type LintResult = Result<(), LintError>; type LintFn = fn() -> LintResult; +struct Linter { + pub description: &'static str, + pub name: &'static str, + pub lint_fn: LintFn, +} + +fn get_linter_list() -> Vec<&'static Linter> { + vec![ + &Linter { + description: "Check that all command line arguments are documented.", + name: "doc", + lint_fn: lint_doc + }, + &Linter { + description: "Check that no symbol from bitcoin-config.h is used without the header being included", + name: "includes_build_config", + lint_fn: lint_includes_build_config + }, + &Linter { + description: "Check that markdown links resolve", + name: "markdown", + lint_fn: lint_markdown + }, + &Linter { + description: "Check that std::filesystem is not used directly", + name: "std_filesystem", + lint_fn: lint_std_filesystem + }, + &Linter { + description: "Check that subtrees are pure subtrees", + name: "subtree", + lint_fn: lint_subtree + }, + &Linter { + description: "Check that tabs are not used as whitespace", + name: "tabs_whitespace", + lint_fn: lint_tabs_whitespace + }, + &Linter { + description: "Check for trailing whitespace", + name: "trailing_whitespace", + lint_fn: lint_trailing_whitespace + }, + &Linter { + description: "Run all linters of the form: test/lint/lint-*.py", + name: "all_python_linters", + lint_fn: run_all_python_linters + }, + ] +} + +fn print_help_and_exit() { + print!( + r#" +Usage: test_runner [--lint=LINTER_TO_RUN] +Runs all linters in the lint test suite, printing any errors +they detect. + +If you wish to only run some particular lint tests, pass +'--lint=' with the name of the lint test you wish to run. +You can set as many '--lint=' values as you wish, e.g.: +test_runner --lint=doc --lint=subtree + +The individual linters available to run are: +"# + ); + for linter in get_linter_list() { + println!("{}: \"{}\"", linter.name, linter.description) + } + + std::process::exit(1); +} + +fn parse_lint_args(args: &[String]) -> Vec<&'static Linter> { + let linter_list = get_linter_list(); + let mut lint_values = Vec::new(); + + for arg in args { + #[allow(clippy::if_same_then_else)] + if arg.starts_with("--lint=") { + let lint_arg_value = arg + .trim_start_matches("--lint=") + .trim_matches('"') + .trim_matches('\''); + + let try_find_linter = linter_list + .iter() + .find(|linter| linter.name == lint_arg_value); + match try_find_linter { + Some(linter) => { + lint_values.push(*linter); + } + None => { + println!("No linter {lint_arg_value} found!"); + print_help_and_exit(); + } + } + } else if arg.eq("--help") || arg.eq("-h") { + print_help_and_exit(); + } else { + print_help_and_exit(); + } + } + + lint_values +} + /// Return the git command fn git() -> Command { let mut git = Command::new("git"); @@ -292,7 +399,52 @@ fn lint_doc() -> LintResult { } } -fn lint_all() -> LintResult { +fn lint_markdown() -> LintResult { + let bin_name = "mlc"; + let mut md_ignore_paths = get_subtrees(); + md_ignore_paths.push("./doc/README_doxygen.md"); + let md_ignore_path_str = md_ignore_paths.join(","); + + let mut cmd = Command::new(bin_name); + cmd.args([ + "--offline", + "--ignore-path", + md_ignore_path_str.as_str(), + "--root-dir", + ".", + ]) + .stdout(Stdio::null()); // Suppress overly-verbose output + + match cmd.output() { + Ok(output) if output.status.success() => Ok(()), + Ok(output) => { + let stderr = String::from_utf8_lossy(&output.stderr); + let filtered_stderr: String = stderr // Filter out this annoying trailing line + .lines() + .filter(|&line| line != "The following links could not be resolved:") + .collect::<Vec<&str>>() + .join("\n"); + Err(format!( + r#" +One or more markdown links are broken. + +Relative links are preferred (but not required) as jumping to file works natively within Emacs. + +Markdown link errors found: +{} + "#, + filtered_stderr + )) + } + Err(e) if e.kind() == ErrorKind::NotFound => { + println!("`mlc` was not found in $PATH, skipping markdown lint check."); + Ok(()) + } + Err(e) => Err(format!("Error running mlc: {}", e)), // Misc errors + } +} + +fn run_all_python_linters() -> LintResult { let mut good = true; let lint_dir = get_git_root().join("test/lint"); for entry in fs::read_dir(lint_dir).unwrap() { @@ -307,7 +459,7 @@ fn lint_all() -> LintResult { .success() { good = false; - println!("^---- failure generated from {}", entry_fn); + println!("^---- ⚠️ Failure generated from {}", entry_fn); } } if good { @@ -318,24 +470,26 @@ fn lint_all() -> LintResult { } fn main() -> ExitCode { - let test_list: Vec<(&str, LintFn)> = vec![ - ("subtree check", lint_subtree), - ("std::filesystem check", lint_std_filesystem), - ("trailing whitespace check", lint_trailing_whitespace), - ("no-tabs check", lint_tabs_whitespace), - ("build config includes check", lint_includes_build_config), - ("-help=1 documentation check", lint_doc), - ("lint-*.py scripts", lint_all), - ]; + let linters_to_run: Vec<&Linter> = if env::args().count() > 1 { + let args: Vec<String> = env::args().skip(1).collect(); + parse_lint_args(&args) + } else { + // If no arguments are passed, run all linters. + get_linter_list() + }; let git_root = get_git_root(); let mut test_failed = false; - for (lint_name, lint_fn) in test_list { + for linter in linters_to_run { // chdir to root before each lint test env::set_current_dir(&git_root).unwrap(); - if let Err(err) = lint_fn() { - println!("{err}\n^---- ⚠️ Failure generated from {lint_name}!"); + if let Err(err) = (linter.lint_fn)() { + println!( + "{err}\n^---- ⚠️ Failure generated from lint check '{}'!", + linter.name + ); + println!("{}", linter.description); test_failed = true; } } diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 482667a26a..9818d73fdf 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -58,6 +58,7 @@ unsigned-integer-overflow:TxConfirmStats::EstimateMedianVal unsigned-integer-overflow:prevector.h unsigned-integer-overflow:EvalScript unsigned-integer-overflow:xoroshiro128plusplus.h +unsigned-integer-overflow:bitset_detail::PopCount implicit-integer-sign-change:CBlockPolicyEstimator::processBlockTx implicit-integer-sign-change:SetStdinEcho implicit-integer-sign-change:compressor.h |
